R Notebook
pulling in DFs and Merging (still needed?)
# #red wines
# Red_wine <- read.csv("wineQualityReds.csv", header=TRUE, sep = ",")
# Red_wine$Type <- 'Red'
#
# #white wines
# White_wine <- read.csv("wineQualityWhites.csv", header=TRUE, sep = ",")
# White_wine$Type <- 'White'
#
# ## consolidated
# Data <- rbind(Red_wine,White_wine)
# drops <- c("X")
# Data <- Data[ , !(names(Data) %in% drops)]
# Data
#
# ##create final DF
# # write.csv(Data,"/Users/colinobrien/Desktop/repo/stats_6021/Stats_project_group_6/Data.csv", row.names = FALSE)
# # write.csv(Data,"/Users/colinobrien/Desktop/repo/stats_6021/Stats_project_group_6/Data", row.names = FALSE)
# ## both the Data and Data csv are the same. I know people prefer one format vs the other so I made both
library(tidyverse)
# library(ROCR)
library(faraway)
library(dplyr)
library(ggplot2)
library(reshape2)
library(leaps)
# install.packages("bestglm")
library(bestglm)
# install.packages("performance")
# library(performance)
knitr::opts_chunk$set(echo = TRUE)
## Load Datasets
full_wines_final <- read.csv("Data_Final.csv", header = TRUE, stringsAsFactors=TRUE)
# Drop quality for simplicity
full_wines_binary_with_qual<-full_wines_final
full_wines_binary <- subset(full_wines_final, select = -c(quality))
## Convert to 0 and 1 for readability
full_wines_binary$cat_quality <- as.integer(full_wines_binary$cat_quality == "High")
set.seed(90210) ##for reproducibility
sample<-sample.int(nrow(full_wines_binary), floor(.80*nrow(full_wines_binary)), replace = F)
train<-full_wines_binary[sample, ] ##training data frame
rownames(train) <- c(1:5197)
test<-full_wines_binary[-sample, ] ##test data frame
## Just for a single boxplot
train_with_qual<-full_wines_binary_with_qual[sample,]
test_with_qual<-full_wines_binary_with_qual[-sample,]
trainEDA
# drops_cats <- c("Type")
# No_cat_train <- train[ , !(names(train) %in% drops_cats)]
# # No_Type
pairs(train, lower.panel = NULL)
# Convert Type to binary to 0 and 1 for correlation
train$Type <- as.integer(train$Type == "White")
test$Type <- as.integer(test$Type == "White")
cor_train <- cor(train)
cor_train fixed.acidity volatile.acidity citric.acid
fixed.acidity 1.00000000 0.21475470 0.3281717780
volatile.acidity 0.21475470 1.00000000 -0.3756083338
citric.acid 0.32817178 -0.37560833 1.0000000000
residual.sugar -0.11435784 -0.19711519 0.1413532761
chlorides 0.30907486 0.38572698 0.0368162603
free.sulfur.dioxide -0.28545891 -0.36296634 0.1459151441
total.sulfur.dioxide -0.32829667 -0.42380894 0.2063508767
density 0.46102677 0.27340816 0.0934734894
pH -0.25053297 0.26430327 -0.3266522997
sulphates 0.31025439 0.23565770 0.0572809414
alcohol -0.09179407 -0.03401873 -0.0006471143
Type -0.48713134 -0.65599571 0.1886904967
cat_quality -0.07066799 -0.26538562 0.0778223488
residual.sugar chlorides free.sulfur.dioxide
fixed.acidity -0.11435784 0.30907486 -0.28545891
volatile.acidity -0.19711519 0.38572698 -0.36296634
citric.acid 0.14135328 0.03681626 0.14591514
residual.sugar 1.00000000 -0.13267263 0.40674512
chlorides -0.13267263 1.00000000 -0.20583891
free.sulfur.dioxide 0.40674512 -0.20583891 1.00000000
total.sulfur.dioxide 0.49459459 -0.29216595 0.71557904
density 0.54686423 0.36564000 0.01712365
pH -0.26561668 0.04191555 -0.15788076
sulphates -0.18182649 0.40162365 -0.19860411
alcohol -0.34842765 -0.25617404 -0.17708863
Type 0.35126769 -0.52157700 0.48355395
cat_quality -0.02120537 -0.18458731 0.04477089
total.sulfur.dioxide density pH
fixed.acidity -0.32829667 0.46102677 -0.25053297
volatile.acidity -0.42380894 0.27340816 0.26430327
citric.acid 0.20635088 0.09347349 -0.32665230
residual.sugar 0.49459459 0.54686423 -0.26561668
chlorides -0.29216595 0.36564000 0.04191555
free.sulfur.dioxide 0.71557904 0.01712365 -0.15788076
total.sulfur.dioxide 1.00000000 0.02373810 -0.24648617
density 0.02373810 1.00000000 0.01687243
pH -0.24648617 0.01687243 1.00000000
sulphates -0.27752789 0.27270901 0.18260830
alcohol -0.26082628 -0.67847127 0.11700658
Type 0.70618430 -0.39538819 -0.33097301
cat_quality -0.04308093 -0.26437860 0.01854094
sulphates alcohol Type cat_quality
fixed.acidity 0.31025439 -0.0917940736 -0.48713134 -0.07066799
volatile.acidity 0.23565770 -0.0340187335 -0.65599571 -0.26538562
citric.acid 0.05728094 -0.0006471143 0.18869050 0.07782235
residual.sugar -0.18182649 -0.3484276488 0.35126769 -0.02120537
chlorides 0.40162365 -0.2561740351 -0.52157700 -0.18458731
free.sulfur.dioxide -0.19860411 -0.1770886318 0.48355395 0.04477089
total.sulfur.dioxide -0.27752789 -0.2608262814 0.70618430 -0.04308093
density 0.27270901 -0.6784712690 -0.39538819 -0.26437860
pH 0.18260830 0.1170065815 -0.33097301 0.01854094
sulphates 1.00000000 -0.0136065691 -0.49215645 0.03195034
alcohol -0.01360657 1.0000000000 0.03945167 0.39668183
Type -0.49215645 0.0394516700 1.00000000 0.12361294
cat_quality 0.03195034 0.3966818275 0.12361294 1.00000000T_F_cor <- abs(cor_train)>.7
T_F_cor fixed.acidity volatile.acidity citric.acid
fixed.acidity TRUE FALSE FALSE
volatile.acidity FALSE TRUE FALSE
citric.acid FALSE FALSE TRUE
residual.sugar FALSE FALSE FALSE
chlorides FALSE FALSE FALSE
free.sulfur.dioxide FALSE FALSE FALSE
total.sulfur.dioxide FALSE FALSE FALSE
density FALSE FALSE FALSE
pH FALSE FALSE FALSE
sulphates FALSE FALSE FALSE
alcohol FALSE FALSE FALSE
Type FALSE FALSE FALSE
cat_quality FALSE FALSE FALSE
residual.sugar chlorides free.sulfur.dioxide
fixed.acidity FALSE FALSE FALSE
volatile.acidity FALSE FALSE FALSE
citric.acid FALSE FALSE FALSE
residual.sugar TRUE FALSE FALSE
chlorides FALSE TRUE FALSE
free.sulfur.dioxide FALSE FALSE TRUE
total.sulfur.dioxide FALSE FALSE TRUE
density FALSE FALSE FALSE
pH FALSE FALSE FALSE
sulphates FALSE FALSE FALSE
alcohol FALSE FALSE FALSE
Type FALSE FALSE FALSE
cat_quality FALSE FALSE FALSE
total.sulfur.dioxide density pH sulphates alcohol
fixed.acidity FALSE FALSE FALSE FALSE FALSE
volatile.acidity FALSE FALSE FALSE FALSE FALSE
citric.acid FALSE FALSE FALSE FALSE FALSE
residual.sugar FALSE FALSE FALSE FALSE FALSE
chlorides FALSE FALSE FALSE FALSE FALSE
free.sulfur.dioxide TRUE FALSE FALSE FALSE FALSE
total.sulfur.dioxide TRUE FALSE FALSE FALSE FALSE
density FALSE TRUE FALSE FALSE FALSE
pH FALSE FALSE TRUE FALSE FALSE
sulphates FALSE FALSE FALSE TRUE FALSE
alcohol FALSE FALSE FALSE FALSE TRUE
Type TRUE FALSE FALSE FALSE FALSE
cat_quality FALSE FALSE FALSE FALSE FALSE
Type cat_quality
fixed.acidity FALSE FALSE
volatile.acidity FALSE FALSE
citric.acid FALSE FALSE
residual.sugar FALSE FALSE
chlorides FALSE FALSE
free.sulfur.dioxide FALSE FALSE
total.sulfur.dioxide TRUE FALSE
density FALSE FALSE
pH FALSE FALSE
sulphates FALSE FALSE
alcohol FALSE FALSE
Type TRUE FALSE
cat_quality FALSE TRUE## create melted
melted_cor_train <- melt(cor_train)
##create heat map Consolidated
ggplot(data = melted_cor_train, aes(x=Var1, y=Var2, fill=value)) +
geom_tile(color = "white")+
scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
theme_minimal()+
theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
coord_fixed()+
labs(title = 'Consolidated (Both Red and White)')
## creating red and white
train_White <- filter(train, Type == 1)
train_Red <- filter(train, Type == 0)
## droping red/white columns
train_White_NoType <- subset(train_White, select = -c(Type))
train_Red_NoType <- subset(train_Red, select = -c(Type))
## creating correlations
cor_train_White_NoType <- cor(train_White_NoType)
cor_train_Red_NoType <- cor(train_Red_NoType)
## melting
melted_cor_train_white <- melt(cor_train_White_NoType)
melted_cor_train_Red <- melt(cor_train_Red_NoType)
##ploting
##create heat map White
ggplot(data = melted_cor_train_white, aes(x=Var1, y=Var2, fill=value)) +
geom_tile(color = "white")+
scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
theme_minimal()+
theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
coord_fixed()+
labs(title = 'White Wine')
##create heat map Red
ggplot(data = melted_cor_train_Red, aes(x=Var1, y=Var2, fill=value)) +
geom_tile(color = "white")+
scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
theme_minimal()+
theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
coord_fixed()+
labs(title = 'Red Wine')
ggplot(data = train, mapping = aes(x=Type)) +
geom_bar()
Regression Testing
## press formula (from class)
get_press <- function(model) {
sum(((model$residuals)/ (1- (lm.influence(model)$hat)))^2)
}## first go
full<-glm(cat_quality~., family=binomial, data=train)
summary(full)
Call:
glm(formula = cat_quality ~ ., family = binomial, data = train)
Deviance Residuals:
Min 1Q Median 3Q Max
-3.4120 -0.8919 0.4303 0.8148 2.6198
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 1.102e+02 4.889e+01 2.253 0.0242 *
fixed.acidity 7.676e-02 5.573e-02 1.377 0.1684
volatile.acidity -4.720e+00 3.291e-01 -14.342 < 2e-16 ***
citric.acid -4.652e-01 2.875e-01 -1.618 0.1057
residual.sugar 1.187e-01 2.104e-02 5.641 1.69e-08 ***
chlorides -1.265e+00 1.195e+00 -1.059 0.2895
free.sulfur.dioxide 1.345e-02 2.878e-03 4.672 2.98e-06 ***
total.sulfur.dioxide -5.753e-03 1.168e-03 -4.924 8.48e-07 ***
density -1.211e+02 4.964e+01 -2.440 0.0147 *
pH 7.375e-01 3.316e-01 2.224 0.0261 *
sulphates 2.096e+00 2.977e-01 7.042 1.89e-12 ***
alcohol 8.500e-01 6.635e-02 12.811 < 2e-16 ***
Type -5.562e-01 2.077e-01 -2.678 0.0074 **
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 6833.2 on 5196 degrees of freedom
Residual deviance: 5335.5 on 5184 degrees of freedom
AIC: 5361.5
Number of Fisher Scoring iterations: 5## removed all insignificant
reduced_1<-glm(formula = cat_quality~volatile.acidity+residual.sugar+free.sulfur.dioxide+total.sulfur.dioxide+density+pH+sulphates+alcohol+Type, family=binomial, data=train)
summary(reduced_1)
Call:
glm(formula = cat_quality ~ volatile.acidity + residual.sugar +
free.sulfur.dioxide + total.sulfur.dioxide + density + pH +
sulphates + alcohol + Type, family = binomial, data = train)
Deviance Residuals:
Min 1Q Median 3Q Max
-3.3471 -0.9020 0.4275 0.8196 2.6625
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 76.416878 31.773373 2.405 0.01617 *
volatile.acidity -4.680054 0.305603 -15.314 < 2e-16 ***
residual.sugar 0.105875 0.014724 7.191 6.44e-13 ***
free.sulfur.dioxide 0.013183 0.002863 4.604 4.15e-06 ***
total.sulfur.dioxide -0.005930 0.001162 -5.101 3.38e-07 ***
density -86.733244 31.675484 -2.738 0.00618 **
pH 0.595611 0.228953 2.601 0.00928 **
sulphates 1.930494 0.285976 6.751 1.47e-11 ***
alcohol 0.894713 0.051370 17.417 < 2e-16 ***
Type -0.523798 0.204855 -2.557 0.01056 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 6833.2 on 5196 degrees of freedom
Residual deviance: 5341.5 on 5187 degrees of freedom
AIC: 5361.5
Number of Fisher Scoring iterations: 5##evaluating model
Reduced1_AIC_train <- reduced_1$aic
##predicted quality for test data based on training data
preds<-predict(reduced_1,newdata=test, type="response")
reduced_1_error <- table(test$cat_quality, preds>0.5)
reduced_1_error
FALSE TRUE
0 240 236
1 121 703evulation_summary <- data.frame(
attempt = 'reduced_1',
AIC = Reduced1_AIC_train,
PRESS = get_press(reduced_1),
'False positive' = round(reduced_1_error[3]/(reduced_1_error[1]+reduced_1_error[3]),3),
'False negative' = round(reduced_1_error[2]/(reduced_1_error[2]+reduced_1_error[4]),3),
'Error Rate' = round((reduced_1_error[2]+reduced_1_error[3])/(reduced_1_error[1]+reduced_1_error[2]+reduced_1_error[3]+reduced_1_error[4]),3)
)
evulation_summarysecond model
https://rstudio-pubs-static.s3.amazonaws.com/2897_9220b21cfc0c43a396ff9abf122bb351.html
# install.packages("bestglm")
## Prepare data
train.for.best.logistic <- within(train, {
y <- cat_quality
})
## Reorder variables
train.for.best.logistic <-
train.for.best.logistic[, c("fixed.acidity","volatile.acidity","citric.acid","residual.sugar","total.sulfur.dioxide","density","chlorides","free.sulfur.dioxide",'pH','sulphates','alcohol','Type',"y")]
## Perform
res.best.logistic <-
bestglm(Xy = train.for.best.logistic,
family = binomial, # binomial family for logistic
IC = "AIC", # Information criteria for
method = "exhaustive")Morgan-Tatar search since family is non-gaussian.res.best.logistic$BestModelssummary(res.best.logistic$BestModel)
Call:
glm(formula = y ~ ., family = family, data = Xi, weights = weights)
Deviance Residuals:
Min 1Q Median 3Q Max
-3.4302 -0.8932 0.4282 0.8157 2.6350
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 1.147e+02 4.864e+01 2.358 0.0184 *
fixed.acidity 8.518e-02 5.513e-02 1.545 0.1223
volatile.acidity -4.763e+00 3.268e-01 -14.574 < 2e-16 ***
citric.acid -5.158e-01 2.834e-01 -1.820 0.0688 .
residual.sugar 1.215e-01 2.085e-02 5.829 5.57e-09 ***
total.sulfur.dioxide -5.688e-03 1.167e-03 -4.876 1.08e-06 ***
density -1.261e+02 4.936e+01 -2.554 0.0106 *
free.sulfur.dioxide 1.332e-02 2.874e-03 4.633 3.61e-06 ***
pH 8.012e-01 3.261e-01 2.457 0.0140 *
sulphates 2.027e+00 2.897e-01 6.998 2.59e-12 ***
alcohol 8.553e-01 6.613e-02 12.934 < 2e-16 ***
Type -5.290e-01 2.059e-01 -2.569 0.0102 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 6833.2 on 5196 degrees of freedom
Residual deviance: 5336.7 on 5185 degrees of freedom
AIC: 5360.7
Number of Fisher Scoring iterations: 5reduced_4 <- res.best.logistic$BestModel
##evaluating model
Reduced4_AIC_train <- reduced_4$aic
##predicted quality for test data based on training data
preds<-predict(reduced_4,newdata=test, type="response")
reduced_4_error <- table(test$cat_quality, preds>0.5)
evulation_summary_4 <- data.frame(
attempt = 'reduced_4_error (all possible)',
AIC = Reduced4_AIC_train,
PRESS = get_press(reduced_4),
'False positive' = round(reduced_4_error[3]/(reduced_4_error[1]+reduced_4_error[3]),3),
'False negative' = round(reduced_4_error[2]/(reduced_4_error[2]+reduced_4_error[4]),3),
'Error Rate' = round((reduced_4_error[2]+reduced_4_error[3])/(reduced_4_error[1]+reduced_4_error[2]+reduced_4_error[3]+reduced_4_error[4]),3)
)
evulation_summary <- rbind(evulation_summary,evulation_summary_4)
# evulation_summary# data.frame(check_collinearity(reduced_4))
#come back and add df stuffin an effort to lower VIFs scores and correlation, I am removing fixed.acidity
reduced_4_2<-glm(cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type, family=binomial, data=train)
summary(reduced_4_2)
Call:
glm(formula = cat_quality ~ volatile.acidity + citric.acid +
residual.sugar + total.sulfur.dioxide + density + free.sulfur.dioxide +
pH + sulphates + alcohol + Type, family = binomial, data = train)
Deviance Residuals:
Min 1Q Median 3Q Max
-3.3469 -0.8997 0.4283 0.8144 2.6767
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 58.609496 33.904591 1.729 0.0839 .
volatile.acidity -4.841743 0.323737 -14.956 < 2e-16 ***
citric.acid -0.434875 0.278458 -1.562 0.1184
residual.sugar 0.099352 0.015302 6.493 8.43e-11 ***
total.sulfur.dioxide -0.005815 0.001165 -4.993 5.94e-07 ***
density -68.496442 33.901654 -2.020 0.0433 *
free.sulfur.dioxide 0.013292 0.002871 4.630 3.66e-06 ***
pH 0.465949 0.243355 1.915 0.0555 .
sulphates 1.962718 0.287251 6.833 8.33e-12 ***
alcohol 0.918353 0.053777 17.077 < 2e-16 ***
Type -0.483884 0.206617 -2.342 0.0192 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 6833.2 on 5196 degrees of freedom
Residual deviance: 5339.0 on 5186 degrees of freedom
AIC: 5361
Number of Fisher Scoring iterations: 5##evaluating model
Reduced4_2_AIC_train <- reduced_4_2$aic
##predicted quality for test data based on training data
preds<-predict(reduced_4_2,newdata=test, type="response")
reduced_4_2_error <- table(test$cat_quality, preds>0.7)
#Curves
evulation_summary_4_2 <- data.frame(
attempt = 'reduced_4_2_error (post VIF adjustments)',
AIC = Reduced4_2_AIC_train,
PRESS = get_press(reduced_4_2),
'False positive' = round(reduced_4_2_error[3]/(reduced_4_2_error[1]+reduced_4_2_error[3]),3),
'False negative' = round(reduced_4_2_error[2]/(reduced_4_2_error[2]+reduced_4_2_error[4]),3),
'Error Rate' = round((reduced_4_2_error[2]+reduced_4_2_error[3])/(reduced_4_2_error[1]+reduced_4_2_error[2]+reduced_4_2_error[3]+reduced_4_2_error[4]),3)
)
evulation_summary <- rbind(evulation_summary,evulation_summary_4_2)
evulation_summaryNA
NAnow looking at outliers (with “best possible”)
summary(reduced_4)
Call:
glm(formula = y ~ ., family = family, data = Xi, weights = weights)
Deviance Residuals:
Min 1Q Median 3Q Max
-3.4302 -0.8932 0.4282 0.8157 2.6350
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 1.147e+02 4.864e+01 2.358 0.0184 *
fixed.acidity 8.518e-02 5.513e-02 1.545 0.1223
volatile.acidity -4.763e+00 3.268e-01 -14.574 < 2e-16 ***
citric.acid -5.158e-01 2.834e-01 -1.820 0.0688 .
residual.sugar 1.215e-01 2.085e-02 5.829 5.57e-09 ***
total.sulfur.dioxide -5.688e-03 1.167e-03 -4.876 1.08e-06 ***
density -1.261e+02 4.936e+01 -2.554 0.0106 *
free.sulfur.dioxide 1.332e-02 2.874e-03 4.633 3.61e-06 ***
pH 8.012e-01 3.261e-01 2.457 0.0140 *
sulphates 2.027e+00 2.897e-01 6.998 2.59e-12 ***
alcohol 8.553e-01 6.613e-02 12.934 < 2e-16 ***
Type -5.290e-01 2.059e-01 -2.569 0.0102 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 6833.2 on 5196 degrees of freedom
Residual deviance: 5336.7 on 5185 degrees of freedom
AIC: 5360.7
Number of Fisher Scoring iterations: 5Now looking into outliers/influence
p <- 12
n <- 5197Cooks
reduced_4_cook <-cooks.distance(reduced_4)
reduced_4_cook[reduced_4_cook>qf(0.5,p,n-p)]named numeric(0)DFFITs
##dffits
DFFITS<-dffits(reduced_4)
DDFFITS_influence <- DFFITS[abs(DFFITS)>2*sqrt(p/n)]
DDFFITS_influence 83 115 133 148 149 171
0.11219627 0.10544184 -0.18411002 -0.09991644 -0.12785114 -0.11374142
183 202 206 225 249 292
0.10867107 -0.16045065 -0.11840913 0.10093853 -0.09841249 0.09816267
309 317 380 455 466 572
0.10078385 0.10471012 -0.11210579 0.09920619 -0.11390124 0.10171119
722 739 747 794 806 912
-0.16019151 0.11513457 0.09736192 -0.10449640 -0.10120073 0.15167391
952 957 1010 1042 1055 1114
0.09704330 -0.16243046 0.11892944 -0.09863713 -0.11344570 -0.10497624
1139 1180 1241 1258 1270 1325
0.10300372 -0.18433803 0.10969510 0.09821539 0.10530848 -0.09678052
1347 1467 1493 1527 1528 1597
-0.10103881 0.11642941 -0.11091048 -0.11561143 -0.12352731 -0.09863589
1620 1658 1693 1776 1846 1873
-0.10722071 -0.09731307 -0.09795498 -0.10237941 0.12561590 -0.12529594
1882 1891 2002 2004 2020 2079
-0.10642853 0.10446926 0.10894528 0.20125104 0.09637786 -0.10426201
2103 2148 2203 2256 2258 2285
0.09853506 -0.27921127 -0.10129823 -0.10012511 0.10312207 0.10896369
2305 2350 2353 2361 2383 2410
0.13774801 0.09853162 -0.15221240 -0.11012001 -0.10852123 -0.12636270
2434 2461 2465 2487 2494 2497
0.12603788 -0.15438799 -0.10048416 0.10125735 -0.12151311 0.10319545
2521 2525 2527 2551 2594 2650
-0.10237941 -0.10131647 -0.10242317 -0.09669244 -0.10722071 -0.10178224
2696 2699 2772 2781 2801 2840
-0.11840913 -0.10221624 0.20008361 -0.10703304 -0.11210579 0.16042062
2852 2892 2948 2965 2978 2994
-0.10974330 0.09732574 -0.09841249 -0.09647583 -0.11806742 0.14196649
2995 2997 3018 3030 3041 3051
0.09920619 -0.13745390 0.10099437 -0.14516539 0.16695970 -0.12151311
3065 3079 3126 3155 3188 3274
-0.21978199 0.10160629 0.17165718 0.13048253 0.17504970 -0.10882370
3282 3330 3347 3367 3378 3381
0.10171119 0.09880694 -0.11538312 -0.11374142 -0.10444247 0.10177520
3496 3526 3557 3564 3571 3709
0.12512066 -0.15993715 -0.13072803 -0.13765634 -0.11317090 -0.11344570
3721 3740 3752 3809 3823 3843
-0.15438799 -0.10404207 -0.15579966 -0.10786452 0.88209659 0.10113003
3848 3895 3922 4024 4035 4065
0.11006220 -0.18174866 -0.12028121 -0.10640867 -0.11091048 0.09637786
4071 4112 4125 4131 4132 4140
-0.31948427 -0.12024521 -0.10057784 -0.10147570 0.17165718 -0.11174867
4260 4300 4401 4425 4463 4490
-0.11267738 0.10499625 -0.10880237 -0.10594437 -0.12634536 -0.09991644
4512 4547 4610 4627 4678 4687
-0.09647583 -0.17220454 0.10741187 -0.21859387 -0.10392610 0.12149435
4783 4790 4794 4829 4873 5045
0.12315874 0.10969510 -0.09878535 -0.11104591 -0.16746164 -0.10529190
5091 5178 5187
-0.10703304 0.11264144 0.10894528 DFBETAs
DFBETAS<-dfbetas(reduced_4)
abs(DFBETAS)>2/sqrt(n) (Intercept) fixed.acidity volatile.acidity citric.acid residual.sugar
1 FALSE FALSE FALSE FALSE FALSE
2 FALSE FALSE FALSE FALSE FALSE
3 FALSE TRUE FALSE TRUE TRUE
4 FALSE FALSE FALSE FALSE FALSE
5 FALSE FALSE FALSE FALSE FALSE
6 FALSE FALSE FALSE FALSE FALSE
7 FALSE FALSE FALSE FALSE FALSE
8 FALSE FALSE FALSE FALSE TRUE
9 FALSE FALSE FALSE FALSE FALSE
10 TRUE TRUE FALSE TRUE TRUE
11 FALSE FALSE FALSE FALSE FALSE
12 FALSE FALSE FALSE FALSE FALSE
13 FALSE FALSE FALSE FALSE FALSE
14 FALSE FALSE FALSE FALSE FALSE
15 FALSE FALSE FALSE FALSE FALSE
16 FALSE FALSE FALSE FALSE FALSE
17 FALSE FALSE FALSE FALSE FALSE
18 FALSE FALSE FALSE FALSE FALSE
19 FALSE FALSE FALSE FALSE FALSE
20 FALSE FALSE FALSE FALSE FALSE
21 FALSE FALSE FALSE TRUE FALSE
22 FALSE FALSE FALSE FALSE FALSE
23 FALSE TRUE FALSE FALSE TRUE
24 FALSE FALSE FALSE FALSE FALSE
25 FALSE TRUE TRUE FALSE FALSE
26 FALSE FALSE FALSE FALSE FALSE
27 FALSE FALSE FALSE FALSE FALSE
28 FALSE FALSE FALSE FALSE FALSE
29 FALSE FALSE FALSE FALSE FALSE
30 FALSE FALSE FALSE FALSE FALSE
31 FALSE FALSE FALSE FALSE FALSE
32 FALSE FALSE FALSE FALSE FALSE
33 FALSE FALSE FALSE FALSE FALSE
34 FALSE FALSE FALSE FALSE FALSE
35 FALSE FALSE FALSE FALSE FALSE
36 FALSE FALSE FALSE FALSE FALSE
37 FALSE FALSE FALSE FALSE FALSE
38 FALSE FALSE FALSE FALSE FALSE
39 FALSE FALSE FALSE FALSE FALSE
40 FALSE FALSE FALSE FALSE FALSE
41 FALSE FALSE TRUE FALSE FALSE
42 FALSE FALSE FALSE FALSE FALSE
43 FALSE FALSE FALSE FALSE FALSE
44 FALSE FALSE FALSE FALSE FALSE
45 FALSE FALSE FALSE FALSE FALSE
46 FALSE FALSE FALSE FALSE FALSE
47 FALSE FALSE FALSE FALSE TRUE
48 FALSE FALSE FALSE TRUE FALSE
49 FALSE FALSE FALSE FALSE FALSE
50 FALSE FALSE FALSE FALSE FALSE
51 FALSE FALSE FALSE FALSE FALSE
52 FALSE FALSE FALSE FALSE FALSE
53 FALSE FALSE FALSE FALSE FALSE
54 FALSE FALSE FALSE FALSE FALSE
55 FALSE FALSE FALSE FALSE FALSE
56 FALSE FALSE FALSE FALSE FALSE
57 FALSE FALSE FALSE FALSE FALSE
58 FALSE FALSE FALSE FALSE FALSE
59 FALSE FALSE TRUE TRUE FALSE
60 FALSE FALSE FALSE FALSE FALSE
61 FALSE FALSE FALSE FALSE FALSE
62 FALSE FALSE FALSE FALSE FALSE
63 FALSE FALSE FALSE FALSE FALSE
64 FALSE FALSE FALSE FALSE FALSE
65 TRUE TRUE FALSE FALSE TRUE
66 FALSE FALSE FALSE FALSE FALSE
67 FALSE FALSE FALSE FALSE FALSE
68 FALSE FALSE FALSE FALSE FALSE
69 FALSE FALSE FALSE FALSE FALSE
70 FALSE FALSE FALSE FALSE FALSE
71 FALSE FALSE FALSE FALSE FALSE
72 FALSE FALSE FALSE FALSE FALSE
73 FALSE FALSE FALSE FALSE FALSE
74 FALSE FALSE FALSE FALSE FALSE
75 FALSE FALSE FALSE FALSE FALSE
76 FALSE FALSE FALSE FALSE FALSE
77 FALSE FALSE FALSE FALSE FALSE
78 FALSE FALSE FALSE FALSE FALSE
79 FALSE FALSE FALSE FALSE FALSE
80 FALSE FALSE FALSE FALSE FALSE
81 FALSE TRUE FALSE FALSE FALSE
82 FALSE FALSE FALSE FALSE FALSE
83 FALSE TRUE TRUE TRUE FALSE
total.sulfur.dioxide density free.sulfur.dioxide pH sulphates
1 FALSE FALSE FALSE FALSE FALSE
2 FALSE FALSE FALSE FALSE FALSE
3 FALSE FALSE FALSE TRUE FALSE
4 FALSE FALSE FALSE FALSE FALSE
5 FALSE FALSE FALSE FALSE FALSE
6 FALSE FALSE FALSE FALSE FALSE
7 FALSE FALSE FALSE FALSE FALSE
8 FALSE FALSE TRUE FALSE TRUE
9 FALSE FALSE FALSE FALSE FALSE
10 TRUE TRUE TRUE FALSE FALSE
11 FALSE FALSE FALSE FALSE FALSE
12 FALSE FALSE FALSE FALSE FALSE
13 FALSE FALSE FALSE FALSE FALSE
14 FALSE FALSE FALSE FALSE FALSE
15 FALSE FALSE FALSE FALSE TRUE
16 FALSE FALSE FALSE FALSE FALSE
17 FALSE FALSE FALSE FALSE TRUE
18 FALSE FALSE FALSE TRUE FALSE
19 FALSE FALSE FALSE FALSE FALSE
20 FALSE FALSE FALSE FALSE FALSE
21 FALSE FALSE FALSE FALSE FALSE
22 FALSE FALSE FALSE FALSE FALSE
23 FALSE FALSE FALSE FALSE FALSE
24 FALSE FALSE FALSE FALSE FALSE
25 FALSE FALSE FALSE FALSE FALSE
26 FALSE FALSE FALSE FALSE FALSE
27 FALSE FALSE FALSE FALSE FALSE
28 FALSE FALSE FALSE FALSE FALSE
29 FALSE FALSE FALSE FALSE FALSE
30 FALSE FALSE FALSE FALSE FALSE
31 FALSE FALSE FALSE FALSE FALSE
32 FALSE FALSE FALSE FALSE FALSE
33 FALSE FALSE FALSE FALSE FALSE
34 FALSE FALSE FALSE FALSE FALSE
35 FALSE FALSE FALSE FALSE FALSE
36 FALSE FALSE FALSE FALSE FALSE
37 FALSE FALSE FALSE FALSE FALSE
38 FALSE FALSE FALSE FALSE FALSE
39 FALSE FALSE FALSE FALSE FALSE
40 FALSE FALSE FALSE FALSE FALSE
41 FALSE FALSE FALSE FALSE TRUE
42 FALSE FALSE FALSE FALSE FALSE
43 FALSE FALSE TRUE FALSE TRUE
44 FALSE FALSE FALSE FALSE FALSE
45 FALSE FALSE FALSE FALSE FALSE
46 FALSE FALSE FALSE FALSE FALSE
47 FALSE FALSE FALSE TRUE FALSE
48 FALSE FALSE FALSE FALSE FALSE
49 FALSE FALSE FALSE FALSE FALSE
50 FALSE FALSE FALSE FALSE FALSE
51 FALSE FALSE FALSE FALSE FALSE
52 FALSE FALSE FALSE FALSE FALSE
53 FALSE FALSE FALSE FALSE FALSE
54 FALSE FALSE FALSE FALSE FALSE
55 FALSE FALSE FALSE FALSE FALSE
56 FALSE FALSE FALSE FALSE FALSE
57 FALSE FALSE FALSE FALSE FALSE
58 FALSE FALSE FALSE FALSE FALSE
59 FALSE FALSE FALSE FALSE FALSE
60 FALSE FALSE FALSE FALSE FALSE
61 FALSE FALSE FALSE TRUE FALSE
62 FALSE FALSE FALSE FALSE FALSE
63 FALSE FALSE FALSE FALSE FALSE
64 FALSE FALSE FALSE FALSE FALSE
65 FALSE TRUE FALSE TRUE FALSE
66 FALSE FALSE FALSE FALSE FALSE
67 FALSE FALSE FALSE FALSE FALSE
68 FALSE FALSE FALSE FALSE FALSE
69 FALSE FALSE FALSE FALSE FALSE
70 FALSE FALSE TRUE FALSE FALSE
71 FALSE FALSE FALSE FALSE FALSE
72 FALSE FALSE FALSE FALSE FALSE
73 FALSE FALSE FALSE FALSE FALSE
74 FALSE FALSE FALSE FALSE FALSE
75 FALSE FALSE FALSE FALSE FALSE
76 FALSE FALSE FALSE FALSE FALSE
77 FALSE FALSE FALSE FALSE FALSE
78 FALSE FALSE FALSE FALSE FALSE
79 FALSE FALSE FALSE FALSE FALSE
80 FALSE FALSE FALSE FALSE FALSE
81 FALSE FALSE FALSE TRUE FALSE
82 FALSE FALSE FALSE FALSE FALSE
83 FALSE FALSE FALSE TRUE FALSE
alcohol Type
1 FALSE FALSE
2 FALSE FALSE
3 FALSE FALSE
4 FALSE FALSE
5 FALSE FALSE
6 FALSE TRUE
7 FALSE FALSE
8 FALSE FALSE
9 FALSE FALSE
10 TRUE TRUE
11 FALSE FALSE
12 FALSE FALSE
13 FALSE FALSE
14 FALSE FALSE
15 FALSE FALSE
16 FALSE FALSE
17 FALSE FALSE
18 FALSE FALSE
19 FALSE FALSE
20 FALSE FALSE
21 FALSE FALSE
22 FALSE FALSE
23 FALSE FALSE
24 FALSE FALSE
25 FALSE FALSE
26 FALSE FALSE
27 FALSE FALSE
28 FALSE FALSE
29 FALSE FALSE
30 FALSE FALSE
31 FALSE FALSE
32 FALSE FALSE
33 FALSE FALSE
34 FALSE FALSE
35 FALSE FALSE
36 FALSE FALSE
37 FALSE FALSE
38 FALSE FALSE
39 FALSE FALSE
40 FALSE FALSE
41 FALSE TRUE
42 FALSE FALSE
43 FALSE FALSE
44 FALSE FALSE
45 FALSE FALSE
46 FALSE FALSE
47 FALSE FALSE
48 FALSE FALSE
49 FALSE FALSE
50 FALSE FALSE
51 FALSE FALSE
52 FALSE FALSE
53 FALSE FALSE
54 FALSE FALSE
55 FALSE FALSE
56 FALSE FALSE
57 FALSE FALSE
58 FALSE FALSE
59 FALSE TRUE
60 FALSE FALSE
61 FALSE FALSE
62 FALSE FALSE
63 FALSE FALSE
64 FALSE FALSE
65 TRUE FALSE
66 FALSE FALSE
67 FALSE FALSE
68 FALSE FALSE
69 FALSE FALSE
70 FALSE FALSE
71 FALSE FALSE
72 FALSE FALSE
73 FALSE FALSE
74 FALSE FALSE
75 FALSE FALSE
76 FALSE FALSE
77 FALSE FALSE
78 FALSE FALSE
79 FALSE FALSE
80 FALSE FALSE
81 FALSE FALSE
82 FALSE FALSE
83 FALSE TRUE
[ reached getOption("max.print") -- omitted 5114 rows ]leverage
##leverages
lev<-lm.influence(reduced_4)$hat
##identify high leverage points
leverages <- lev[lev>2*p/n]
leverages 25 48 53 65 133 145
0.004849640 0.007980011 0.007028687 0.005763404 0.022975591 0.005173793
171 175 187 202 225 228
0.005219474 0.004875721 0.006161515 0.005447858 0.006756723 0.005384983
248 249 255 282 299 309
0.008357256 0.004801962 0.005867337 0.005674515 0.005484537 0.010250038
332 333 339 345 361 380
0.004754212 0.004686940 0.004990699 0.005924282 0.006389325 0.006435227
383 417 427 455 466 489
0.006875256 0.004943798 0.004867913 0.009165489 0.006894948 0.007172970
516 557 600 607 708 722
0.010652503 0.005554653 0.005521417 0.005900775 0.004918820 0.005283197
730 747 749 765 770 806
0.005147385 0.005767084 0.005739296 0.005627857 0.004853531 0.005540805
828 873 899 912 945 957
0.007828147 0.006995196 0.005410461 0.009841709 0.005463074 0.011974020
965 995 1010 1037 1055 1114
0.005623028 0.014123202 0.004921736 0.006212390 0.006141910 0.005265677
1132 1139 1140 1152 1169 1176
0.006966068 0.005959266 0.006430522 0.006059376 0.004777701 0.005844926
1180 1188 1241 1255 1265 1300
0.012489770 0.004953075 0.004856461 0.004777701 0.005672314 0.006081786
1304 1306 1325 1345 1375 1427
0.005100154 0.005735362 0.005574241 0.004711974 0.006383030 0.004930082
1467 1479 1481 1482 1493 1495
0.006002690 0.006720302 0.005242150 0.014123202 0.007969759 0.004698112
1507 1518 1527 1528 1612 1614
0.004854829 0.005124711 0.005382525 0.005299431 0.009822894 0.008993399
1638 1642 1649 1667 1711 1719
0.007179770 0.007952008 0.005823824 0.007088896 0.004814463 0.004889259
1768 1801 1829 1845 1846 1873
0.004851478 0.004848096 0.004766974 0.006026022 0.005410441 0.005408285
1882 1887 1891 1894 1929 1933
0.004657424 0.005524176 0.005158734 0.004960904 0.007033838 0.006340479
1935 1941 1956 1979 2002 2004
0.005367322 0.005155488 0.004781888 0.006023785 0.005565063 0.013293498
2019 2097 2106 2120 2135 2137
0.005674515 0.004945639 0.005391888 0.005466349 0.006032961 0.005458302
2144 2148 2155 2165 2177 2203
0.005593131 0.020382883 0.005008204 0.005477897 0.005419422 0.004636068
2211 2218 2232 2270 2281 2294
0.004669295 0.019421769 0.006338349 0.007220018 0.006915245 0.006786840
2305 2306 2317 2350 2353 2361
0.010603689 0.005150988 0.011839638 0.004828941 0.012864370 0.005000134
2371 2379 2383 2391 2396 2400
0.005408758 0.005477511 0.006524792 0.005514256 0.008839432 0.007088896
2410 2420 2434 2461 2469 2494
0.007468644 0.004680633 0.020272029 0.016139547 0.005146048 0.008984307
2497 2525 2602 2623 2650 2663
0.006080078 0.006378263 0.007252053 0.006726745 0.005745347 0.006470764
2680 2688 2699 2772 2777 2779
0.007158112 0.005672314 0.007016065 0.013464526 0.012846622 0.006384951
2798 2801 2812 2840 2849 2852
0.004748024 0.006435227 0.007487716 0.008115731 0.006556965 0.004882351
2856 2900 2910 2912 2946 2948
0.006384951 0.005156491 0.006091693 0.004787344 0.005054188 0.004801962
2965 2969 2971 2990 2994 2995
0.005054630 0.005674515 0.004891501 0.005016018 0.005494005 0.009165489
3004 3008 3012 3018 3021 3030
0.004821646 0.005616506 0.005184523 0.007585533 0.007383966 0.008916163
3041 3046 3051 3052 3065 3074
0.016455429 0.005893423 0.008984307 0.005310204 0.012183875 0.004946746
3126 3130 3155 3188 3221 3274
0.019492710 0.005384397 0.007093095 0.015836977 0.005940855 0.005995139
3292 3309 3324 3332 3347 3367
0.004646160 0.005018396 0.005633797 0.008055287 0.007555507 0.005219474
3378 3381 3413 3418 3419 3446
0.005844088 0.007215437 0.007952008 0.004995318 0.005468012 0.004726109
3469 3474 3487 3496 3519 3524
0.005565027 0.004674916 0.005068929 0.009369520 0.005543214 0.004635661
3526 3545 3549 3557 3587 3611
0.013422322 0.004757291 0.004918820 0.007684917 0.004872856 0.004872856
3622 3650 3666 3709 3710 3720
0.004733307 0.005857163 0.006405776 0.006141910 0.005615658 0.006389325
3721 3727 3741 3752 3763 3769
0.016139547 0.006618658 0.004870547 0.016330934 0.005287558 0.004728145
3787 3792 3807 3818 3820 3823
0.004943798 0.007689718 0.005998108 0.010775791 0.007366569 0.323058874
3824 3835 3843 3848 3854 3863
0.012258040 0.004967080 0.005005760 0.006333501 0.007689718 0.008161154
3890 3895 3912 3917 3922 3953
0.005616506 0.019354101 0.004912963 0.007690767 0.010025916 0.019421769
3991 4024 4035 4071 4088 4112
0.004912256 0.005151978 0.007969759 0.033246417 0.004918863 0.007559978
4125 4129 4131 4132 4140 4143
0.005498808 0.004805200 0.007131721 0.019492710 0.009613280 0.004686940
4174 4180 4191 4195 4260 4287
0.006136951 0.005524176 0.005514261 0.005146048 0.009759507 0.004801486
4300 4301 4343 4362 4401 4463
0.005225384 0.005310204 0.007165155 0.005507166 0.005566348 0.017712588
4482 4493 4511 4512 4547 4553
0.008767404 0.007154999 0.006995196 0.005054630 0.009347506 0.010326676
4558 4585 4627 4633 4639 4687
0.006875101 0.004851478 0.014885235 0.005546699 0.004777701 0.006998472
4693 4696 4710 4715 4754 4759
0.004640983 0.005081857 0.005440630 0.005196371 0.006386644 0.007418423
4790 4829 4873 4902 4908 4972
0.004856461 0.006130203 0.012769233 0.005893423 0.008892413 0.004912256
4992 5040 5041 5054 5087 5107
0.006275606 0.004889259 0.007383966 0.006838011 0.004875721 0.006541276
5154 5178 5187
0.007677790 0.006034090 0.005565063 outlier
reduced_4.res <- reduced_4$residuals
crit<-qt(1-0.05/(2*n), n-p-1)
outliers <- reduced_4.res[abs(reduced_4.res)>crit]
outliers 8 12 21 30 83 85
-5.913799 -112.810937 -5.131140 -28.388375 7.064053 -4.902421
86 101 115 122 142 148
5.268800 -6.738462 5.835978 -11.281066 -4.900743 -5.950818
149 154 166 183 202 206
-8.284466 -4.614577 5.536043 18.759111 -11.064846 -6.976696
218 245 264 281 288 372
7.906909 -7.166119 6.562536 4.514517 -9.756502 -9.170360
373 440 505 509 525 548
-4.811386 -4.829656 10.431685 -5.199469 32.192103 -42.498339
566 589 629 655 657 659
10.705696 -4.616565 -5.057653 4.903313 -10.049213 -4.454203
675 687 701 706 717 721
-5.471326 -6.529182 -9.150344 -8.992792 -7.819105 -8.206050
722 725 739 751 768 774
-11.839559 -6.756563 4.469453 9.211716 -5.691134 -4.977692
788 794 795 817 819 833
5.293240 -5.395176 -9.673765 -13.711257 -4.723057 -10.325351
840 844 875 888 902 903
-4.977692 7.896471 -13.765446 4.511825 -4.486556 4.815690
917 954 964 967 991 1039
-9.997107 11.645455 -5.057653 7.233862 -4.851860 -29.863426
1077 1081 1107 1124 1133 1156
-6.559467 -5.979445 -11.538933 -9.743607 -8.597773 -358.992053
1162 1167 1171 1172 1226 1235
-5.913799 -13.370172 -5.991466 -13.904782 -24.686598 -22.516690
1257 1258 1263 1270 1305 1318
9.644487 4.622980 6.011786 8.044160 -5.145541 -5.100743
1344 1356 1387 1412 1428 1436
5.658642 -6.254700 -4.851860 -9.750690 -11.498443 -8.316040
1487 1497 1510 1524 1610 1620
7.687631 5.004046 -7.060659 4.482981 -16.343618 -5.238351
1652 1658 1665 1669 1693 1706
-11.418067 -7.852930 -5.163340 -16.484320 -4.689106 -4.877840
1776 1787 1790 1840 1930 1951
-16.318931 -5.613721 9.189242 -4.798258 -6.274087 -7.515991
1958 1998 2004 2006 2018 2040
-6.682392 9.251057 4.599440 5.240035 7.500553 4.734365
2041 2059 2079 2115 2119 2148
-5.264313 -15.336442 -6.043512 -23.388238 -4.859254 -6.605338
2156 2167 2195 2205 2220 2256
-5.287846 -9.600193 -5.857549 -4.579628 4.948918 -9.053372
2258 2262 2284 2290 2313 2329
17.356606 -7.264594 -16.484320 5.684278 -4.851860 -7.041619
2334 2335 2337 2386 2439 2452
-7.417245 -4.555771 -5.816303 -12.954904 9.264406 -8.282663
2480 2485 2506 2521 2536 2579
-4.793832 -18.780311 4.541019 -16.318931 -4.884719 9.211716
2594 2628 2629 2642 2660 2661
-5.238351 -6.381604 4.489202 -5.096416 4.453651 -5.461202
2666 2696 2702 2718 2727 2735
-6.254700 -6.976696 7.233862 -6.726264 5.198906 -6.197960
2745 2763 2772 2781 2793 2795
-6.606402 -11.892705 4.431016 -8.362063 -4.697738 -4.530220
2802 2840 2866 2892 2898 2955
-7.010108 4.976958 -7.349414 11.007314 -4.528817 -14.101843
2978 2994 2997 3005 3025 3040
-13.426497 6.463923 -14.095802 5.659028 -4.737794 -8.881971
3042 3065 3072 3079 3090 3105
-6.738462 -7.313679 -6.423514 16.073667 5.085934 -5.548543
3133 3149 3196 3210 3243 3248
-5.466040 -5.426271 -6.355608 -7.747739 -6.033627 5.580542
3257 3264 3322 3336 3343 3351
-4.595999 -5.857549 -9.308640 9.264406 -4.501913 5.867902
3362 3369 3400 3410 3414 3424
-13.219174 -12.293034 -4.896657 -5.318501 -9.993826 -6.001601
3434 3461 3522 3564 3566 3571
-10.863890 -6.353654 -8.866347 -8.604661 -6.119343 -6.889088
3582 3605 3629 3638 3711 3716
-5.322449 -9.260519 -8.348111 -6.028213 -6.506898 5.779253
3740 3750 3784 3816 3841 3853
-7.586267 5.060560 -4.746179 -6.432015 -6.569981 5.240035
3855 3866 3903 3910 3913 3915
-10.826648 -4.935304 5.739331 -9.056519 -4.882192 -16.323382
3942 3947 3951 4003 4019 4090
-5.403135 -4.485039 -6.092031 -8.881971 -4.877840 -4.542401
4093 4111 4116 4127 4144 4145
-5.913799 -15.362967 -5.909070 -7.854482 4.541019 -8.716350
4151 4156 4159 4170 4233 4243
-9.593344 -7.055209 -42.578664 -4.837952 -4.614577 -5.850980
4248 4306 4311 4329 4354 4356
-14.114484 -5.627580 4.678171 -11.261020 -14.672621 -5.417642
4460 4490 4503 4505 4527 4544
6.046933 -5.950818 -6.716718 4.453651 4.477563 -8.992792
4547 4556 4580 4610 4614 4627
-4.960846 -4.813642 -13.243536 5.353589 -5.833289 -4.965536
4650 4656 4659 4677 4701 4713
-4.896657 4.644296 -6.288973 5.012168 -10.498461 8.507914
4729 4783 4794 4815 4858 4880
-6.726264 7.580155 -5.470226 -5.334876 -5.096416 -6.738462
4887 4910 4924 4939 4988 5045
5.612130 -6.821823 -4.543092 4.488823 -6.821823 -11.196377
5049 5051 5091 5095 5133 5140
-4.972743 -6.197392 -8.362063 -7.809710 -5.399943 4.815690
5179 5195
-8.541941 -10.162346 ## outliers removed
outliers_index <- attr(outliers, "names")
outliers_index <- as.numeric(outliers_index)
train_no_outliers <- train[-(outliers_index),]
#leverages removed
lererages_index <- attr(leverages, "names")
lererages_index <- as.numeric(lererages_index)
train_no_leverages <- train[-(lererages_index),]
# DDFFITS_influence
DDFFITS_index <- attr(DDFFITS_influence, "names")
DDFFITS_index <- as.numeric(DDFFITS_index)
train_no_DDFFITS <- train[-(DDFFITS_index),]
# all "non-normal" removed
all_special <- c(DDFFITS_index,lererages_index,outliers_index)
train_nothing_special <- train[-(all_special),]
train_nothing_specialNA
NA
NA
NAvif(train[c(2,3,4,7,8,6,9,10,11)]) volatile.acidity citric.acid residual.sugar
1.818602 1.505238 3.353752
total.sulfur.dioxide density free.sulfur.dioxide
2.850601 5.664883 2.099196
pH sulphates alcohol
1.338240 1.427424 2.890129 train_temp<-train
# as.factor(train_temp$Type)<-numeric(train_temp$Type)
#train_temp
# as.factor
train_temp$Type <- as.numeric(train_temp$Type)-1
train_temp$Type <- as.integer(train_temp$Type)
train_tempcreating reduced
reduced_4_3 <- glm(cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type, family=binomial, data=train_no_outliers)
reduced_4_4_lev <- glm(cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type, family=binomial, data=train_no_leverages)
reduced_4_5_DDFFITS <- glm(cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type, family=binomial, data=train_no_DDFFITS)
reduced_4_6_no_special <- glm(cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type, family=binomial, data=train_nothing_special)
# summary(reduced_4_6_no_special)
## checking colinearity / VIF scores
# reduced_4_3_col <- data.frame('reduced_4_3' = check_collinearity(reduced_4_3))
# reduced_4_3_col_VIF <- reduced_4_3_col[c('reduced_4_3.Term','reduced_4_3.VIF')]
# reduced_4_3_col_VIF
#
# reduced_4_4_lev_col <- data.frame('reduced_4_4_lev' = check_collinearity(reduced_4_4_lev))
# reduced_4_4_lev_col_VIF <- reduced_4_4_lev_col[c('reduced_4_4_lev.Term','reduced_4_4_lev.VIF')]
# reduced_4_4_lev_col_VIF
#
#
# reduced_4_5_DDFFITS_col <- data.frame('reduced_4_5_DDFFITS' = check_collinearity(reduced_4_5_DDFFITS))
# reduced_4_5_DDFFITS_col_VIF <- reduced_4_5_DDFFITS_col[c('reduced_4_5_DDFFITS.Term','reduced_4_5_DDFFITS.VIF')]
# reduced_4_5_DDFFITS_col_VIF
#
# reduced_4_6_no_special_col <- data.frame('reduced_4_6_no_special' = check_collinearity(reduced_4_6_no_special))
# reduced_4_6_no_special_col_VIF <- reduced_4_6_no_special_col[c('reduced_4_6_no_special.Term','reduced_4_6_no_special.VIF')]
# reduced_4_6_no_special_col_VIF
#
# VIF_summary <- data.frame('0'=reduced_4_3_col_VIF['reduced_4_3.Term'],
# '1'=reduced_4_3_col_VIF['reduced_4_3.VIF'],
# '2'=reduced_4_4_lev_col_VIF['reduced_4_4_lev.VIF'],
# '3'=reduced_4_5_DDFFITS_col_VIF['reduced_4_5_DDFFITS.VIF'],
# '4'=reduced_4_6_no_special_col_VIF['reduced_4_6_no_special.VIF'])
# colnames(VIF_summary) <- c('Predictor Variable','4_3.VIF.Outliers','4_4_lev.VIF','4_5_DDFFITS.VIF','4_6_no_special.VIF')
# VIF_summary
## VIF for Outliers
### cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type
#
reg_4_VIF_test <- vif(train_temp[c(1,2,3,4,7,8,6,9,10,11,12)])
reg_4_2_VIF_test <- vif(train_temp[c(2,3,4,7,8,6,9,10,11,12)])
outliers_VIF <- vif(train_no_outliers[c(2,3,4,7,8,6,9,10,11,12)])
leverage_VIF <- vif(train_no_leverages[c(2,3,4,7,8,6,9,10,11,12)])
DDFFITS_VIF <- vif(train_no_DDFFITS[c(2,3,4,7,8,6,9,10,11,12)])
nothing_special <- vif(train_nothing_special[c(2,3,4,7,8,6,9,10,11,12)])
reg_4_VIF_test fixed.acidity volatile.acidity citric.acid residual.sugar total.sulfur.dioxide
4.962487 2.139008 1.585795 9.280982 3.974170
density free.sulfur.dioxide pH sulphates alcohol
21.338984 2.189173 2.483275 1.514014 5.371286
Type
7.045601 VIF_summary_test <- data.frame('best_possible_VIF (post)'=reg_4_2_VIF_test,
'outliers_VIF'=outliers_VIF,
'leverage_VIF'=leverage_VIF,
'DDFFITS_VIF'= DDFFITS_VIF,
'nothing_special'=nothing_special)
VIF_summary_testNA##evaluating model
Reduced4_3_AIC_train <- reduced_4_3$aic
##predicted quality for test data based on training data
preds<-predict(reduced_4_3,newdata=test, type="response")
reduced_4_3_error <- table(test$cat_quality, preds>0.6)
evulation_summary_4_3 <- data.frame(
attempt = 'reduced_4_3_error_outliers',
AIC = Reduced4_3_AIC_train,
PRESS = get_press(reduced_4_3),
'False positive' = round(reduced_4_3_error[3]/(reduced_4_3_error[1]+reduced_4_3_error[3]),3),
'False negative' = round(reduced_4_3_error[2]/(reduced_4_3_error[2]+reduced_4_3_error[4]),3),
'Error Rate' = round((reduced_4_3_error[2]+reduced_4_3_error[3])/(reduced_4_3_error[1]+reduced_4_3_error[2]+reduced_4_3_error[3]+reduced_4_3_error[4]),3)
)
evulation_summary <- rbind(evulation_summary,evulation_summary_4_3)
evulation_summary##evaluating model leverage
reduced_4_4_lev_AIC_train <- reduced_4_4_lev$aic
##predicted quality for test data based on training data
preds<-predict(reduced_4_4_lev,newdata=test, type="response")
reduced_4_4_lev_error <- table(test$cat_quality, preds>0.65)
evulation_summary_4_4_lev <- data.frame(
attempt = 'reduced_4_4_lev_error',
AIC = reduced_4_4_lev_AIC_train,
PRESS = get_press(reduced_4_4_lev),
'False positive' = round(reduced_4_4_lev_error[3]/(reduced_4_4_lev_error[1]+reduced_4_4_lev_error[3]),3),
'False negative' = round(reduced_4_4_lev_error[2]/(reduced_4_4_lev_error[2]+reduced_4_4_lev_error[4]),3),
'Error Rate' = round((reduced_4_4_lev_error[2]+reduced_4_4_lev_error[3])/(reduced_4_4_lev_error[1]+reduced_4_4_lev_error[2]+reduced_4_4_lev_error[3]+reduced_4_4_lev_error[4]),3)
)
evulation_summary <- rbind(evulation_summary,evulation_summary_4_4_lev)
evulation_summary##evaluating model DDFFITS
reduced_4_5_DDFFITS_AIC_train <- reduced_4_5_DDFFITS$aic
##predicted quality for test data based on training data
preds<-predict(reduced_4_5_DDFFITS,newdata=test, type="response")
reduced_4_5_DDFFITS_error <- table(test$cat_quality, preds>0.7)
evulation_summary_4_5_DDFFITS <- data.frame(
attempt = 'reduced_4_5_DDFFITS_error',
AIC = reduced_4_5_DDFFITS_AIC_train,
PRESS = get_press(reduced_4_5_DDFFITS),
'False positive' = round(reduced_4_5_DDFFITS_error[3]/(reduced_4_5_DDFFITS_error[1]+reduced_4_5_DDFFITS_error[3]),3),
'False negative' = round(reduced_4_5_DDFFITS_error[2]/(reduced_4_5_DDFFITS_error[2]+reduced_4_5_DDFFITS_error[4]),3),
'Error Rate' = round((reduced_4_5_DDFFITS_error[2]+reduced_4_5_DDFFITS_error[3])/(reduced_4_5_DDFFITS_error[1]+reduced_4_5_DDFFITS_error[2]+reduced_4_5_DDFFITS_error[3]+reduced_4_5_DDFFITS_error[4]),3)
)
evulation_summary <- rbind(evulation_summary,evulation_summary_4_5_DDFFITS)
evulation_summary##evaluating model DDFFITS
reduced_4_6_no_special_AIC_train <- reduced_4_6_no_special$aic
##predicted quality for test data based on training data
preds<-predict(reduced_4_6_no_special,newdata=test, type="response")
reduced_4_6_no_special_error <- table(test$cat_quality, preds>0.8)
evulation_summary_4_6_no_special <- data.frame(
attempt = 'reduced_4_6_no_special_error',
AIC = reduced_4_6_no_special_AIC_train,
PRESS = get_press(reduced_4_6_no_special),
'False positive' = round(reduced_4_6_no_special_error[3]/(reduced_4_6_no_special_error[1]+reduced_4_6_no_special_error[3]),3),
'False negative' = round(reduced_4_6_no_special_error[2]/(reduced_4_6_no_special_error[2]+reduced_4_6_no_special_error[4]),3),
'Error Rate' = round((reduced_4_6_no_special_error[2]+reduced_4_6_no_special_error[3])/(reduced_4_6_no_special_error[1]+reduced_4_6_no_special_error[2]+reduced_4_6_no_special_error[3]+reduced_4_6_no_special_error[4]),3)
)
evulation_summary <- rbind(evulation_summary,evulation_summary_4_6_no_special)
evulation_summaryROC Curves and AUC
## reduced_1
# detach(package:performance, unload=TRUE)
## FYI the performance package causes ROC curves to not work
library(ROCR)
# reduced_1
preds<-predict(reduced_1,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_1")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
reduced_1_auc <- auc@y.values
## reduced_4
preds<-predict(reduced_4,newdata=test, type="response")
rates4<-prediction(preds, test$cat_quality)
roc_result<-performance(rates4,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4")
lines(x = c(0,1), y = c(0,1), col="red")
auc4<-performance(rates4, measure = "auc")
reduced_4_auc <- auc4@y.values
## reduced_4_2
preds<-predict(reduced_4_2,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_2")
lines(x = c(0,1), y = c(0,1), col="red")
auc4_2<-performance(rates, measure = "auc")
reduced_4_2_auc <- auc4_2@y.values
## reduced_4_3
preds<-predict(reduced_4_3,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_3")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
reduced_4_3_auc <- auc@y.values
## reduced_4_4_lev
preds<-predict(reduced_4_4_lev,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_4_lev")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
reduced_4_4_lev_auc <- auc@y.values
## reduced_4_5_DDFFITS
preds<-predict(reduced_4_5_DDFFITS,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_5_DDFFITS")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
reduced_4_5_DDFFITS_auc <- auc@y.values
## reduced_4_6_no_special
preds<-predict(reduced_4_6_no_special,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_6_no_special")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
reduced_4_6_no_special_auc <- auc@y.values
AUC_summary <- data.frame('reduced_1'=reduced_1_auc,
'reduced_4'=reduced_4_auc,
'reduced_4_2'=reduced_4_2_auc,
'reduced_4_3'=reduced_4_3_auc,
'reduced_4_4_lev'=reduced_4_4_lev_auc,
'reduced_4_5_DDFFITS'=reduced_4_5_DDFFITS_auc,
'reduced_4_6_no_special'=reduced_4_6_no_special_auc)
colnames(AUC_summary) <- c('reduced_1','reduced_4','reduced_4_2','reduced_4_3','reduced_4_4_lev','reduced_4_5_DDFFITS','reduced_4_6_no_special')
AUC_summaryRyan’s part starts here
The goal is to make 3 models: One for just white, one for just red, and one with interaction terms with the type of wine.
After that, the models will be trained on the filtered datasets and the resulting scores will be added to the evaluation summary.
Red wine only model
regfull_Red<-glm(cat_quality~., family="binomial", data=train_Red_NoType)
regnull_Red<-glm(cat_quality~1, family="binomial", data=train_Red_NoType)
step(regnull_Red, scope=list(lower=regnull_Red, upper=regfull_Red), direction="forward")Start: AIC=1811.89
cat_quality ~ 1
Df Deviance AIC
+ alcohol 1 1540.6 1544.6
+ volatile.acidity 1 1680.6 1684.6
+ total.sulfur.dioxide 1 1726.9 1730.9
+ sulphates 1 1741.1 1745.1
+ citric.acid 1 1778.7 1782.7
+ density 1 1780.0 1784.0
+ chlorides 1 1795.4 1799.4
+ fixed.acidity 1 1797.8 1801.8
+ free.sulfur.dioxide 1 1803.8 1807.8
<none> 1809.9 1811.9
+ pH 1 1809.8 1813.8
+ residual.sugar 1 1809.9 1813.9
Step: AIC=1544.61
cat_quality ~ alcohol
Df Deviance AIC
+ volatile.acidity 1 1450.7 1456.7
+ sulphates 1 1492.7 1498.7
+ total.sulfur.dioxide 1 1507.4 1513.4
+ fixed.acidity 1 1521.6 1527.6
+ citric.acid 1 1523.5 1529.5
+ pH 1 1528.7 1534.7
+ density 1 1535.8 1541.8
<none> 1540.6 1544.6
+ free.sulfur.dioxide 1 1539.2 1545.2
+ residual.sugar 1 1539.8 1545.8
+ chlorides 1 1540.5 1546.5
Step: AIC=1456.66
cat_quality ~ alcohol + volatile.acidity
Df Deviance AIC
+ total.sulfur.dioxide 1 1417.1 1425.1
+ sulphates 1 1427.5 1435.5
+ fixed.acidity 1 1448.1 1456.1
+ free.sulfur.dioxide 1 1448.1 1456.1
<none> 1450.7 1456.7
+ citric.acid 1 1449.0 1457.0
+ density 1 1449.5 1457.5
+ residual.sugar 1 1449.8 1457.8
+ pH 1 1449.9 1457.9
+ chlorides 1 1450.7 1458.7
Step: AIC=1425.14
cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide
Df Deviance AIC
+ sulphates 1 1387.4 1397.4
+ free.sulfur.dioxide 1 1409.8 1419.8
<none> 1417.1 1425.1
+ pH 1 1415.8 1425.8
+ density 1 1416.1 1426.1
+ fixed.acidity 1 1416.2 1426.2
+ citric.acid 1 1416.8 1426.8
+ residual.sugar 1 1417.0 1427.0
+ chlorides 1 1417.1 1427.1
Step: AIC=1397.37
cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide +
sulphates
Df Deviance AIC
+ free.sulfur.dioxide 1 1379.6 1391.6
+ chlorides 1 1380.0 1392.0
+ citric.acid 1 1383.9 1395.9
<none> 1387.4 1397.4
+ residual.sugar 1 1387.1 1399.1
+ fixed.acidity 1 1387.3 1399.3
+ pH 1 1387.4 1399.4
+ density 1 1387.4 1399.4
Step: AIC=1391.64
cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide +
sulphates + free.sulfur.dioxide
Df Deviance AIC
+ chlorides 1 1372.6 1386.6
<none> 1379.6 1391.6
+ citric.acid 1 1377.9 1391.9
+ pH 1 1379.3 1393.3
+ fixed.acidity 1 1379.3 1393.3
+ residual.sugar 1 1379.5 1393.5
+ density 1 1379.6 1393.6
Step: AIC=1386.63
cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide +
sulphates + free.sulfur.dioxide + chlorides
Df Deviance AIC
<none> 1372.6 1386.6
+ pH 1 1371.2 1387.2
+ citric.acid 1 1372.1 1388.1
+ fixed.acidity 1 1372.2 1388.2
+ residual.sugar 1 1372.2 1388.2
+ density 1 1372.6 1388.6
Call: glm(formula = cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide +
sulphates + free.sulfur.dioxide + chlorides, family = "binomial",
data = train_Red_NoType)
Coefficients:
(Intercept) alcohol volatile.acidity
-8.02155 0.85512 -2.93115
total.sulfur.dioxide sulphates free.sulfur.dioxide
-0.01843 2.63739 0.02326
chlorides
-4.10141
Degrees of Freedom: 1308 Total (i.e. Null); 1302 Residual
Null Deviance: 1810
Residual Deviance: 1373 AIC: 1387The model looks great after the foward selection! Time to test and add to the evaluation summary.
model1_Red<-glm(formula = cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide +
sulphates + free.sulfur.dioxide + chlorides, family = "binomial",
data = train_Red_NoType)
summary(model1_Red)
Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide +
sulphates + free.sulfur.dioxide + chlorides, family = "binomial",
data = train_Red_NoType)
Deviance Residuals:
Min 1Q Median 3Q Max
-3.0655 -0.8628 0.3239 0.8474 2.3082
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) -8.021553 0.886912 -9.044 < 2e-16 ***
alcohol 0.855123 0.078409 10.906 < 2e-16 ***
volatile.acidity -2.931148 0.413395 -7.090 1.34e-12 ***
total.sulfur.dioxide -0.018428 0.002926 -6.297 3.03e-10 ***
sulphates 2.637391 0.456973 5.771 7.86e-09 ***
free.sulfur.dioxide 0.023257 0.008608 2.702 0.00689 **
chlorides -4.101411 1.588464 -2.582 0.00982 **
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 1809.9 on 1308 degrees of freedom
Residual deviance: 1372.6 on 1302 degrees of freedom
AIC: 1386.6
Number of Fisher Scoring iterations: 4
##evaluating model
model1_Red_AIC_train <- model1_Red$aic
##predicted quality for test data based on training data
test_Red_NoType<-subset(test, Type == 0, select=-c(Type))
preds<-predict(model1_Red,newdata=test_Red_NoType, type="response")
model1_Red_error <- table(test_Red_NoType$cat_quality, preds>0.7)
#Curves
evulation_summary_1R <- data.frame(
attempt = 'model1_Red',
AIC = model1_Red_AIC_train,
PRESS = get_press(model1_Red),
'False positive' = round(model1_Red_error[3]/(model1_Red_error[1]+model1_Red_error[3]),3),
'False negative' = round(model1_Red_error[2]/(model1_Red_error[2]+model1_Red_error[4]),3),
'Error Rate' = round((model1_Red_error[2]+model1_Red_error[3])/(model1_Red_error[1]+model1_Red_error[2]+model1_Red_error[3]+model1_Red_error[4]),3)
)
compare_models<-rbind(evulation_summary[1,],evulation_summary_1R)
compare_models
evulation_summary <- rbind(evulation_summary,evulation_summary_1R)
evulation_summaryNA# model1_Red
library(ROCR)
preds<-predict(model1_Red,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_Red")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
model1_Red_auc <- auc@y.values
White wine only model
regfull_White<-glm(cat_quality~., family="binomial", data=train_White_NoType)
regnull_White<-glm(cat_quality~1,family="binomial", data=train_White_NoType)
step(regnull_White, scope=list(lower=regnull_White, upper=regfull_White), direction="forward")Start: AIC=4947.36
cat_quality ~ 1
Df Deviance AIC
+ alcohol 1 4280.3 4284.3
+ density 1 4668.2 4672.2
+ volatile.acidity 1 4758.7 4762.7
+ chlorides 1 4805.4 4809.4
+ total.sulfur.dioxide 1 4831.9 4835.9
+ fixed.acidity 1 4912.0 4916.0
+ pH 1 4914.3 4918.3
+ residual.sugar 1 4919.1 4923.1
+ sulphates 1 4934.7 4938.7
<none> 4945.4 4947.4
+ free.sulfur.dioxide 1 4945.2 4949.2
+ citric.acid 1 4945.4 4949.4
Step: AIC=4284.29
cat_quality ~ alcohol
Df Deviance AIC
+ volatile.acidity 1 4030.1 4036.1
+ residual.sugar 1 4214.8 4220.8
+ free.sulfur.dioxide 1 4235.1 4241.1
+ density 1 4253.7 4259.7
+ sulphates 1 4261.3 4267.3
+ fixed.acidity 1 4267.3 4273.3
+ chlorides 1 4273.1 4279.1
+ pH 1 4276.2 4282.2
+ citric.acid 1 4276.3 4282.3
<none> 4280.3 4284.3
+ total.sulfur.dioxide 1 4279.4 4285.4
Step: AIC=4036.1
cat_quality ~ alcohol + volatile.acidity
Df Deviance AIC
+ residual.sugar 1 3942.1 3950.1
+ density 1 3984.6 3992.6
+ free.sulfur.dioxide 1 3999.3 4007.3
+ sulphates 1 4013.2 4021.2
+ fixed.acidity 1 4014.8 4022.8
+ total.sulfur.dioxide 1 4020.2 4028.2
<none> 4030.1 4036.1
+ pH 1 4028.3 4036.3
+ chlorides 1 4028.6 4036.6
+ citric.acid 1 4030.1 4038.1
Step: AIC=3950.11
cat_quality ~ alcohol + volatile.acidity + residual.sugar
Df Deviance AIC
+ fixed.acidity 1 3923.0 3933.0
+ sulphates 1 3924.2 3934.2
+ free.sulfur.dioxide 1 3930.2 3940.2
+ density 1 3932.9 3942.9
+ pH 1 3934.3 3944.3
<none> 3942.1 3950.1
+ total.sulfur.dioxide 1 3941.1 3951.1
+ citric.acid 1 3941.6 3951.6
+ chlorides 1 3942.1 3952.1
Step: AIC=3932.95
cat_quality ~ alcohol + volatile.acidity + residual.sugar + fixed.acidity
Df Deviance AIC
+ sulphates 1 3904.7 3916.7
+ free.sulfur.dioxide 1 3914.0 3926.0
<none> 3923.0 3933.0
+ pH 1 3921.5 3933.5
+ total.sulfur.dioxide 1 3921.7 3933.7
+ density 1 3921.9 3933.9
+ citric.acid 1 3922.7 3934.7
+ chlorides 1 3923.0 3935.0
Step: AIC=3916.74
cat_quality ~ alcohol + volatile.acidity + residual.sugar + fixed.acidity +
sulphates
Df Deviance AIC
+ free.sulfur.dioxide 1 3896.9 3910.9
+ density 1 3898.2 3912.2
<none> 3904.7 3916.7
+ pH 1 3904.6 3918.6
+ total.sulfur.dioxide 1 3904.6 3918.6
+ citric.acid 1 3904.7 3918.7
+ chlorides 1 3904.7 3918.7
Step: AIC=3910.91
cat_quality ~ alcohol + volatile.acidity + residual.sugar + fixed.acidity +
sulphates + free.sulfur.dioxide
Df Deviance AIC
+ density 1 3890.7 3906.7
+ total.sulfur.dioxide 1 3894.7 3910.7
<none> 3896.9 3910.9
+ pH 1 3896.8 3912.8
+ chlorides 1 3896.9 3912.9
+ citric.acid 1 3896.9 3912.9
Step: AIC=3906.7
cat_quality ~ alcohol + volatile.acidity + residual.sugar + fixed.acidity +
sulphates + free.sulfur.dioxide + density
Df Deviance AIC
+ pH 1 3885.2 3903.2
<none> 3890.7 3906.7
+ total.sulfur.dioxide 1 3889.6 3907.6
+ chlorides 1 3890.7 3908.7
+ citric.acid 1 3890.7 3908.7
Step: AIC=3903.24
cat_quality ~ alcohol + volatile.acidity + residual.sugar + fixed.acidity +
sulphates + free.sulfur.dioxide + density + pH
Df Deviance AIC
<none> 3885.2 3903.2
+ total.sulfur.dioxide 1 3884.3 3904.3
+ chlorides 1 3885.0 3905.0
+ citric.acid 1 3885.1 3905.1
Call: glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar +
fixed.acidity + sulphates + free.sulfur.dioxide + density +
pH, family = "binomial", data = train_White_NoType)
Coefficients:
(Intercept) alcohol volatile.acidity
2.041e+02 8.487e-01 -6.416e+00
residual.sugar fixed.acidity sulphates
1.563e-01 -9.407e-03 1.877e+00
free.sulfur.dioxide density pH
6.778e-03 -2.165e+02 8.920e-01
Degrees of Freedom: 3887 Total (i.e. Null); 3879 Residual
Null Deviance: 4945
Residual Deviance: 3885 AIC: 3903The model looks good after the foward selection, but the predictor fixed.acidity can be removed. The density VIF is above ten, but jsut barely. For now, it will be left in. Time to test and add to the evaluation summary.
model1_White<-glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar +
fixed.acidity + sulphates + free.sulfur.dioxide + density +
pH, family = "binomial", data = train_White_NoType)
summary(model1_White)
Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar +
fixed.acidity + sulphates + free.sulfur.dioxide + density +
pH, family = "binomial", data = train_White_NoType)
Deviance Residuals:
Min 1Q Median 3Q Max
-3.198 -0.888 0.437 0.798 2.507
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 2.041e+02 6.993e+01 2.918 0.00352 **
alcohol 8.487e-01 9.686e-02 8.763 < 2e-16 ***
volatile.acidity -6.416e+00 4.477e-01 -14.329 < 2e-16 ***
residual.sugar 1.563e-01 2.746e-02 5.689 1.27e-08 ***
fixed.acidity -9.407e-03 7.582e-02 -0.124 0.90126
sulphates 1.877e+00 4.028e-01 4.661 3.15e-06 ***
free.sulfur.dioxide 6.778e-03 2.540e-03 2.669 0.00761 **
density -2.165e+02 7.088e+01 -3.054 0.00226 **
pH 8.920e-01 3.886e-01 2.295 0.02172 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 4945.4 on 3887 degrees of freedom
Residual deviance: 3885.2 on 3879 degrees of freedom
AIC: 3903.2
Number of Fisher Scoring iterations: 5model1_White<-glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar + sulphates + free.sulfur.dioxide + density +
pH, family = "binomial", data = train_White_NoType)
summary(model1_White)
Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar +
sulphates + free.sulfur.dioxide + density + pH, family = "binomial",
data = train_White_NoType)
Deviance Residuals:
Min 1Q Median 3Q Max
-3.1956 -0.8867 0.4377 0.7961 2.5070
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 2.105e+02 4.762e+01 4.420 9.87e-06 ***
alcohol 8.408e-01 7.311e-02 11.501 < 2e-16 ***
volatile.acidity -6.407e+00 4.417e-01 -14.504 < 2e-16 ***
residual.sugar 1.586e-01 1.972e-02 8.045 8.66e-16 ***
sulphates 1.885e+00 3.978e-01 4.739 2.14e-06 ***
free.sulfur.dioxide 6.796e-03 2.536e-03 2.680 0.00737 **
density -2.230e+02 4.773e+01 -4.673 2.97e-06 ***
pH 9.247e-01 2.859e-01 3.235 0.00122 **
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 4945.4 on 3887 degrees of freedom
Residual deviance: 3885.3 on 3880 degrees of freedom
AIC: 3901.3
Number of Fisher Scoring iterations: 5
##evaluating model
model1_White_AIC_train <- model1_White$aic
##predicted quality for test data based on training data
test_White_NoType<-subset(test, Type == 1, select=-c(Type))
preds<-predict(model1_White,newdata=test_White_NoType, type="response")
model1_White_error <- table(test_White_NoType$cat_quality, preds>0.7)
#Curves
evulation_summary_1W <- data.frame(
attempt = 'model1_White',
AIC = model1_White_AIC_train,
PRESS = get_press(model1_White),
'False positive' = round(model1_White_error[3]/(model1_White_error[1]+model1_White_error[3]),3),
'False negative' = round(model1_White_error[2]/(model1_White_error[2]+model1_White_error[4]),3),
'Error Rate' = round((model1_White_error[2]+model1_White_error[3])/(model1_White_error[1]+model1_White_error[2]+model1_White_error[3]+model1_White_error[4]),3)
)
evulation_summary <- rbind(evulation_summary,evulation_summary_1W)
evulation_summary
compare_models<-rbind(compare_models,evulation_summary_1W)
compare_modelsNAThe model with interaction terms
regfull_int<-glm(cat_quality~.*Type, family="binomial", data=train)
regnull_int<-glm(cat_quality~1,family="binomial", data=train)
step(regnull_int, scope=list(lower=regnull_int, upper=regfull_int), direction="forward")Start: AIC=6835.15
cat_quality ~ 1
Df Deviance AIC
+ alcohol 1 5904.3 5908.3
+ density 1 6450.6 6454.6
+ volatile.acidity 1 6468.6 6472.6
+ chlorides 1 6640.0 6644.0
+ Type 1 6755.3 6759.3
+ citric.acid 1 6801.3 6805.3
+ fixed.acidity 1 6807.6 6811.6
+ free.sulfur.dioxide 1 6822.7 6826.7
+ total.sulfur.dioxide 1 6823.5 6827.5
+ sulphates 1 6827.8 6831.8
+ residual.sugar 1 6830.8 6834.8
<none> 6833.2 6835.2
+ pH 1 6831.4 6835.4
Step: AIC=5908.3
cat_quality ~ alcohol
Df Deviance AIC
+ volatile.acidity 1 5539.7 5545.7
+ residual.sugar 1 5781.4 5787.4
+ free.sulfur.dioxide 1 5810.8 5816.8
+ Type 1 5826.6 5832.6
+ chlorides 1 5861.3 5867.3
+ citric.acid 1 5864.8 5870.8
+ total.sulfur.dioxide 1 5872.3 5878.3
+ fixed.acidity 1 5893.0 5899.0
+ pH 1 5895.5 5901.5
+ sulphates 1 5897.6 5903.6
<none> 5904.3 5908.3
+ density 1 5903.5 5909.5
Step: AIC=5545.67
cat_quality ~ alcohol + volatile.acidity
Df Deviance AIC
+ density 1 5476.5 5484.5
+ sulphates 1 5477.1 5485.1
+ residual.sugar 1 5500.5 5508.5
+ Type 1 5504.4 5512.4
+ total.sulfur.dioxide 1 5524.4 5532.4
+ pH 1 5532.9 5540.9
+ free.sulfur.dioxide 1 5533.7 5541.7
<none> 5539.7 5545.7
+ chlorides 1 5537.8 5545.8
+ citric.acid 1 5537.9 5545.9
+ fixed.acidity 1 5538.6 5546.6
Step: AIC=5484.51
cat_quality ~ alcohol + volatile.acidity + density
Df Deviance AIC
+ sulphates 1 5442.1 5452.1
+ citric.acid 1 5463.1 5473.1
+ fixed.acidity 1 5463.8 5473.8
+ total.sulfur.dioxide 1 5464.0 5474.0
+ Type 1 5465.2 5475.2
+ pH 1 5471.0 5481.0
+ free.sulfur.dioxide 1 5471.2 5481.2
+ residual.sugar 1 5472.8 5482.8
<none> 5476.5 5484.5
+ chlorides 1 5476.3 5486.3
Step: AIC=5452.14
cat_quality ~ alcohol + volatile.acidity + density + sulphates
Df Deviance AIC
+ residual.sugar 1 5419.3 5431.3
+ fixed.acidity 1 5422.1 5434.1
+ citric.acid 1 5424.1 5436.1
+ free.sulfur.dioxide 1 5432.7 5444.7
+ total.sulfur.dioxide 1 5435.6 5447.6
+ pH 1 5439.3 5451.3
+ chlorides 1 5439.4 5451.4
<none> 5442.1 5452.1
+ Type 1 5440.5 5452.5
Step: AIC=5431.3
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar
Df Deviance AIC
+ Type 1 5379.2 5393.2
+ total.sulfur.dioxide 1 5385.9 5399.9
+ citric.acid 1 5404.8 5418.8
+ pH 1 5410.7 5424.7
+ fixed.acidity 1 5415.4 5429.4
<none> 5419.3 5431.3
+ free.sulfur.dioxide 1 5417.5 5431.5
+ chlorides 1 5418.8 5432.8
Step: AIC=5393.24
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type
Df Deviance AIC
+ volatile.acidity:Type 1 5348.7 5364.7
+ total.sulfur.dioxide 1 5371.4 5387.4
+ pH 1 5371.9 5387.9
+ citric.acid 1 5372.7 5388.7
+ free.sulfur.dioxide 1 5373.9 5389.9
+ residual.sugar:Type 1 5374.6 5390.6
+ chlorides 1 5376.4 5392.4
+ density:Type 1 5377.1 5393.1
<none> 5379.2 5393.2
+ fixed.acidity 1 5378.4 5394.4
+ alcohol:Type 1 5379.2 5395.2
+ sulphates:Type 1 5379.2 5395.2
Step: AIC=5364.67
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + volatile.acidity:Type
Df Deviance AIC
+ total.sulfur.dioxide 1 5342.0 5360.0
+ residual.sugar:Type 1 5343.3 5361.3
+ free.sulfur.dioxide 1 5344.5 5362.5
+ pH 1 5344.7 5362.7
+ density:Type 1 5345.2 5363.2
+ chlorides 1 5345.8 5363.8
+ citric.acid 1 5346.2 5364.2
<none> 5348.7 5364.7
+ alcohol:Type 1 5348.3 5366.3
+ sulphates:Type 1 5348.5 5366.5
+ fixed.acidity 1 5348.6 5366.6
Step: AIC=5359.98
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + total.sulfur.dioxide + volatile.acidity:Type
Df Deviance AIC
+ total.sulfur.dioxide:Type 1 5303.1 5323.1
+ free.sulfur.dioxide 1 5323.1 5343.1
+ residual.sugar:Type 1 5337.0 5357.0
+ pH 1 5337.3 5357.3
+ chlorides 1 5338.9 5358.9
+ density:Type 1 5339.4 5359.4
+ citric.acid 1 5339.9 5359.9
<none> 5342.0 5360.0
+ alcohol:Type 1 5341.7 5361.7
+ fixed.acidity 1 5341.8 5361.8
+ sulphates:Type 1 5341.9 5361.9
Step: AIC=5323.07
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + total.sulfur.dioxide + volatile.acidity:Type +
Type:total.sulfur.dioxide
Df Deviance AIC
+ free.sulfur.dioxide 1 5286.9 5308.9
+ density:Type 1 5295.6 5317.6
+ pH 1 5299.6 5321.6
+ alcohol:Type 1 5300.2 5322.2
+ chlorides 1 5300.3 5322.3
<none> 5303.1 5323.1
+ citric.acid 1 5302.1 5324.1
+ sulphates:Type 1 5302.2 5324.2
+ residual.sugar:Type 1 5302.4 5324.4
+ fixed.acidity 1 5302.9 5324.9
Step: AIC=5308.91
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide +
volatile.acidity:Type + Type:total.sulfur.dioxide
Df Deviance AIC
+ density:Type 1 5281.0 5305.0
+ chlorides 1 5283.6 5307.6
+ alcohol:Type 1 5283.8 5307.8
+ pH 1 5284.1 5308.1
<none> 5286.9 5308.9
+ free.sulfur.dioxide:Type 1 5285.6 5309.6
+ citric.acid 1 5285.8 5309.8
+ residual.sugar:Type 1 5286.2 5310.2
+ sulphates:Type 1 5286.3 5310.3
+ fixed.acidity 1 5286.8 5310.8
Step: AIC=5305.04
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide +
volatile.acidity:Type + Type:total.sulfur.dioxide + density:Type
Df Deviance AIC
+ pH 1 5275.6 5301.6
+ residual.sugar:Type 1 5276.7 5302.7
+ chlorides 1 5277.1 5303.1
<none> 5281.0 5305.0
+ citric.acid 1 5279.1 5305.1
+ free.sulfur.dioxide:Type 1 5279.2 5305.2
+ fixed.acidity 1 5280.0 5306.0
+ alcohol:Type 1 5280.6 5306.6
+ sulphates:Type 1 5280.9 5306.9
Step: AIC=5301.61
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide +
pH + volatile.acidity:Type + Type:total.sulfur.dioxide +
density:Type
Df Deviance AIC
+ residual.sugar:Type 1 5270.3 5298.3
+ pH:Type 1 5271.9 5299.9
+ chlorides 1 5272.9 5300.9
<none> 5275.6 5301.6
+ free.sulfur.dioxide:Type 1 5274.3 5302.3
+ fixed.acidity 1 5275.0 5303.0
+ citric.acid 1 5275.1 5303.1
+ sulphates:Type 1 5275.2 5303.2
+ alcohol:Type 1 5275.4 5303.4
Step: AIC=5298.31
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide +
pH + volatile.acidity:Type + Type:total.sulfur.dioxide +
density:Type + residual.sugar:Type
Df Deviance AIC
+ pH:Type 1 5266.7 5296.7
+ chlorides 1 5268.0 5298.0
<none> 5270.3 5298.3
+ free.sulfur.dioxide:Type 1 5268.4 5298.4
+ fixed.acidity 1 5269.7 5299.7
+ citric.acid 1 5269.9 5299.9
+ sulphates:Type 1 5270.1 5300.1
+ alcohol:Type 1 5270.2 5300.2
Step: AIC=5296.67
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide +
pH + volatile.acidity:Type + Type:total.sulfur.dioxide +
density:Type + residual.sugar:Type + Type:pH
Df Deviance AIC
+ chlorides 1 5263.4 5295.4
+ free.sulfur.dioxide:Type 1 5263.8 5295.8
<none> 5266.7 5296.7
+ citric.acid 1 5265.9 5297.9
+ alcohol:Type 1 5266.4 5298.4
+ sulphates:Type 1 5266.6 5298.6
+ fixed.acidity 1 5266.6 5298.6
Step: AIC=5295.44
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide +
pH + chlorides + volatile.acidity:Type + Type:total.sulfur.dioxide +
density:Type + residual.sugar:Type + Type:pH
Df Deviance AIC
+ chlorides:Type 1 5259.3 5293.3
+ free.sulfur.dioxide:Type 1 5260.5 5294.5
<none> 5263.4 5295.4
+ sulphates:Type 1 5262.7 5296.7
+ citric.acid 1 5263.0 5297.0
+ alcohol:Type 1 5263.2 5297.2
+ fixed.acidity 1 5263.4 5297.4
Step: AIC=5293.31
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide +
pH + chlorides + volatile.acidity:Type + Type:total.sulfur.dioxide +
density:Type + residual.sugar:Type + Type:pH + Type:chlorides
Df Deviance AIC
+ free.sulfur.dioxide:Type 1 5256.2 5292.2
<none> 5259.3 5293.3
+ sulphates:Type 1 5257.9 5293.9
+ citric.acid 1 5258.9 5294.9
+ alcohol:Type 1 5259.3 5295.3
+ fixed.acidity 1 5259.3 5295.3
Step: AIC=5292.16
cat_quality ~ alcohol + volatile.acidity + density + sulphates +
residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide +
pH + chlorides + volatile.acidity:Type + Type:total.sulfur.dioxide +
density:Type + residual.sugar:Type + Type:pH + Type:chlorides +
Type:free.sulfur.dioxide
Df Deviance AIC
<none> 5256.2 5292.2
+ sulphates:Type 1 5254.7 5292.7
+ citric.acid 1 5255.9 5293.9
+ alcohol:Type 1 5256.1 5294.1
+ fixed.acidity 1 5256.2 5294.2
Call: glm(formula = cat_quality ~ alcohol + volatile.acidity + density +
sulphates + residual.sugar + Type + total.sulfur.dioxide +
free.sulfur.dioxide + pH + chlorides + volatile.acidity:Type +
Type:total.sulfur.dioxide + density:Type + residual.sugar:Type +
Type:pH + Type:chlorides + Type:free.sulfur.dioxide, family = "binomial",
data = train)
Coefficients:
(Intercept) alcohol
9.68713 0.84990
volatile.acidity density
-2.84978 -15.58322
sulphates residual.sugar
2.24908 0.03117
Type total.sulfur.dioxide
195.20143 -0.01898
free.sulfur.dioxide pH
0.02478 -0.60200
chlorides volatile.acidity:Type
-4.02470 -3.50882
Type:total.sulfur.dioxide density:Type
0.01756 -201.96697
residual.sugar:Type Type:pH
0.12808 1.50862
Type:chlorides Type:free.sulfur.dioxide
5.10775 -0.01620
Degrees of Freedom: 5196 Total (i.e. Null); 5179 Residual
Null Deviance: 6833
Residual Deviance: 5256 AIC: 5292The forward step process dropped + sulphates:Type, fixed.acidity, alcohol:Type, and citric.acid By the hierarchical principle, the two non-interactive terms need to be added back because their have interaction terms are in the model.
model1_int<-glm(formula = cat_quality ~ alcohol + volatile.acidity + density +
sulphates + residual.sugar + Type + total.sulfur.dioxide +
free.sulfur.dioxide + pH + chlorides + volatile.acidity:Type +
Type:total.sulfur.dioxide + density:Type + residual.sugar:Type +
Type:pH + Type:chlorides + Type:free.sulfur.dioxide, family = "binomial",
data = train)
summary(model1_int)
Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + density +
sulphates + residual.sugar + Type + total.sulfur.dioxide +
free.sulfur.dioxide + pH + chlorides + volatile.acidity:Type +
Type:total.sulfur.dioxide + density:Type + residual.sugar:Type +
Type:pH + Type:chlorides + Type:free.sulfur.dioxide, family = "binomial",
data = train)
Deviance Residuals:
Min 1Q Median 3Q Max
-3.2378 -0.8774 0.4148 0.8069 2.4950
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 9.687e+00 4.539e+01 0.213 0.830982
alcohol 8.499e-01 5.665e-02 15.002 < 2e-16 ***
volatile.acidity -2.850e+00 4.243e-01 -6.716 1.87e-11 ***
density -1.558e+01 4.495e+01 -0.347 0.728824
sulphates 2.249e+00 3.032e-01 7.418 1.19e-13 ***
residual.sugar 3.117e-02 5.020e-02 0.621 0.534702
Type 1.952e+02 5.575e+01 3.501 0.000463 ***
total.sulfur.dioxide -1.897e-02 2.917e-03 -6.505 7.77e-11 ***
free.sulfur.dioxide 2.478e-02 8.602e-03 2.880 0.003971 **
pH -6.020e-01 4.860e-01 -1.239 0.215502
chlorides -4.025e+00 1.527e+00 -2.636 0.008391 **
volatile.acidity:Type -3.509e+00 6.200e-01 -5.659 1.52e-08 ***
Type:total.sulfur.dioxide 1.756e-02 3.200e-03 5.487 4.09e-08 ***
density:Type -2.020e+02 5.584e+01 -3.617 0.000299 ***
residual.sugar:Type 1.281e-01 5.214e-02 2.456 0.014036 *
Type:pH 1.509e+00 5.610e-01 2.689 0.007159 **
Type:chlorides 5.108e+00 2.438e+00 2.095 0.036133 *
Type:free.sulfur.dioxide -1.620e-02 9.147e-03 -1.771 0.076642 .
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 6833.2 on 5196 degrees of freedom
Residual deviance: 5256.2 on 5179 degrees of freedom
AIC: 5292.2
Number of Fisher Scoring iterations: 5
##evaluating model
model1_int_AIC_train <- model1_int$aic
##predicted quality for test data based on training data
preds<-predict(model1_int,newdata=test, type="response")
model1_int_error <- table(test$cat_quality, preds>0.7)
#Curves
evulation_summary_1int <- data.frame(
attempt = 'model1_int',
AIC = model1_int_AIC_train,
PRESS = get_press(model1_int),
'False positive' = round(model1_int_error[3]/(model1_int_error[1]+model1_int_error[3]),3),
'False negative' = round(model1_int_error[2]/(model1_int_error[2]+model1_int_error[4]),3),
'Error Rate' = round((model1_int_error[2]+model1_int_error[3])/(model1_int_error[1]+model1_int_error[2]+model1_int_error[3]+model1_int_error[4]),3)
)
evulation_summary <- rbind(evulation_summary,evulation_summary_1int)
evulation_summary
compare_models<-rbind(compare_models,evulation_summary_1int)
compare_modelsNAcompare_models<-compare_models%>%
rename(
Model = attempt
)
library(data.table)Registered S3 method overwritten by 'data.table':
method from
print.data.table
data.table 1.14.0 using 1 threads (see ?getDTthreads). Latest news: r-datatable.com
**********
This installation of data.table has not detected OpenMP support. It should still work but in single-threaded mode.
This is a Mac. Please read https://mac.r-project.org/openmp/. Please engage with Apple and ask them for support. Check r-datatable.com for updates, and our Mac instructions here: https://github.com/Rdatatable/data.table/wiki/Installation. After several years of many reports of installation problems on Mac, it's time to gingerly point out that there have been no similar problems on Windows or Linux.
**********
Attaching package: ‘data.table’
The following objects are masked from ‘package:reshape2’:
dcast, melt
The following objects are masked from ‘package:dplyr’:
between, first, last
The following object is masked from ‘package:purrr’:
transposelibrary(dplyr)
library(formattable)Registered S3 methods overwritten by 'htmltools':
method from
print.html tools:rstudio
print.shiny.tag tools:rstudio
print.shiny.tag.list tools:rstudio
Registered S3 method overwritten by 'htmlwidgets':
method from
print.htmlwidget tools:rstudiolibrary(tidyr)
customGreen0 = "#DeF7E9"
customGreen = "#71CA97"
customRed = "#ff7f7f"
formattable(compare_models,align =c("l","c", "c", "c", "c", "r"))
NACreating the ROC curves and AUC for the 3 new models.
# model1_Red
preds<-predict(model1_Red,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_Red")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
model1_Red_auc <- auc@y.values
# model1_White
preds<-predict(model1_White,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_White")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
model1_White_auc <- auc@y.values
# model1_int
preds<-predict(model1_int,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_int")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
model1_int_auc <- auc@y.valuesThis is the one liners that run the tables and figures!
##create heat map Consolidated
ggplot(data = melted_cor_train, aes(x=Var1, y=Var2, fill=value)) +
geom_tile(color = "white")+
scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
theme_minimal()+
theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
coord_fixed()+
labs(title = 'Consolidated (Both Red and White)')
##create heat map White
ggplot(data = melted_cor_train_white, aes(x=Var1, y=Var2, fill=value)) +
geom_tile(color = "white")+
scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
theme_minimal()+
theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
coord_fixed()+
labs(title = 'White Wine')
##create heat map Red
ggplot(data = melted_cor_train_Red, aes(x=Var1, y=Var2, fill=value)) +
geom_tile(color = "white")+
scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
theme_minimal()+
theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
coord_fixed()+
labs(title = 'Red Wine')
ggplot(train_with_qual, mapping = aes(x=quality, fill=Type))+
geom_histogram(binwidth=1, alpha=.4, position="identity", color="black")+
geom_vline(aes(xintercept=5.5, color="red"),
linetype="dashed")+
scale_color_manual(name = "Cut Off", values = c("red"))+
labs(x="Quality",
y="Frequency",
title="Distribution of Quality Rating by Wine Type")
This is the table for showing the evaluation for the first model
formattable(evulation_summary[1,])summary(model1_Red)summary(model1_White)summary(model1_int)
Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + density +
sulphates + residual.sugar + Type + total.sulfur.dioxide +
free.sulfur.dioxide + pH + chlorides + volatile.acidity:Type +
Type:total.sulfur.dioxide + density:Type + residual.sugar:Type +
Type:pH + Type:chlorides + Type:free.sulfur.dioxide, family = "binomial",
data = train)
Deviance Residuals:
Min 1Q Median 3Q Max
-3.2378 -0.8774 0.4148 0.8069 2.4950
Coefficients:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 9.687e+00 4.539e+01 0.213 0.830982
alcohol 8.499e-01 5.665e-02 15.002 < 2e-16 ***
volatile.acidity -2.850e+00 4.243e-01 -6.716 1.87e-11 ***
density -1.558e+01 4.495e+01 -0.347 0.728824
sulphates 2.249e+00 3.032e-01 7.418 1.19e-13 ***
residual.sugar 3.117e-02 5.020e-02 0.621 0.534702
Type 1.952e+02 5.575e+01 3.501 0.000463 ***
total.sulfur.dioxide -1.897e-02 2.917e-03 -6.505 7.77e-11 ***
free.sulfur.dioxide 2.478e-02 8.602e-03 2.880 0.003971 **
pH -6.020e-01 4.860e-01 -1.239 0.215502
chlorides -4.025e+00 1.527e+00 -2.636 0.008391 **
volatile.acidity:Type -3.509e+00 6.200e-01 -5.659 1.52e-08 ***
Type:total.sulfur.dioxide 1.756e-02 3.200e-03 5.487 4.09e-08 ***
density:Type -2.020e+02 5.584e+01 -3.617 0.000299 ***
residual.sugar:Type 1.281e-01 5.214e-02 2.456 0.014036 *
Type:pH 1.509e+00 5.610e-01 2.689 0.007159 **
Type:chlorides 5.108e+00 2.438e+00 2.095 0.036133 *
Type:free.sulfur.dioxide -1.620e-02 9.147e-03 -1.771 0.076642 .
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Dispersion parameter for binomial family taken to be 1)
Null deviance: 6833.2 on 5196 degrees of freedom
Residual deviance: 5256.2 on 5179 degrees of freedom
AIC: 5292.2
Number of Fisher Scoring iterations: 5Table right above the “Best Possible Model (Reduced_4)” section.
formattable(compare_models,align =c("l","c", "c", "c", "c", "r"))
This is the table for showing the best models (top five)
formattable(res.best.logistic$BestModels)Add ROC for reduced_1, model1_Red, model1_White, model1_int
preds<-predict(reduced_1,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_1")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
reduced_1_auc <- auc@y.values
# model1_Red
preds<-predict(model1_Red,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_Red")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
model1_Red_auc <- auc@y.values
# model1_White
preds<-predict(model1_White,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_White")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
model1_White_auc <- auc@y.values
# model1_int
preds<-predict(model1_int,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_int")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
model1_int_auc <- auc@y.valuesThis is the table for showing the best models (top five)
formattable(evulation_summary[2,])This is the table for reduced_4 VIF.
formattable(data.frame(reg_4_VIF_test), align =c("l","r"))This is the next VIF plot in the report
formattable(data.frame(reg_4_2_VIF_test), align =c("l","r"))The table below that. It is the evaluation summary for reduced_4_2
formattable(evulation_summary[3,])evaluation summary for the outlier/leverage/etc.
formattable(evulation_summary[4:7,])add roc curves for these four.
## reduced_4
preds<-predict(reduced_4,newdata=test, type="response")Error: variable 'Type' was fitted with type "numeric" but type "factor" was supplied
## reduced_4_2
preds<-predict(reduced_4_2,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_2")
lines(x = c(0,1), y = c(0,1), col="red")
auc4_2<-performance(rates, measure = "auc")
reduced_4_2_auc <- auc4_2@y.values
## reduced_4_3
preds<-predict(reduced_4_3,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_3")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
reduced_4_3_auc <- auc@y.values
## reduced_4_4_lev
preds<-predict(reduced_4_4_lev,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_4_lev")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
reduced_4_4_lev_auc <- auc@y.values
## reduced_4_5_DDFFITS
preds<-predict(reduced_4_5_DDFFITS,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_5_DDFFITS")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
reduced_4_5_DDFFITS_auc <- auc@y.values
## reduced_4_6_no_special
preds<-predict(reduced_4_6_no_special,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_6_no_special")
lines(x = c(0,1), y = c(0,1), col="red")
auc<-performance(rates, measure = "auc")
reduced_4_6_no_special_auc <- auc@y.values
AUC_summary <- data.frame('reduced_1'=reduced_1_auc,
'reduced_4'=reduced_4_auc,
'reduced_4_2'=reduced_4_2_auc,
'reduced_4_3'=reduced_4_3_auc,
'reduced_4_4_lev'=reduced_4_4_lev_auc,
'reduced_4_5_DDFFITS'=reduced_4_5_DDFFITS_auc,
'reduced_4_6_no_special'=reduced_4_6_no_special_auc)
colnames(AUC_summary) <- c('reduced_1','reduced_4','reduced_4_2','reduced_4_3','reduced_4_4_lev','reduced_4_5_DDFFITS','reduced_4_6_no_special')
AUC_summaryLS0tCnRpdGxlOiAiUiBOb3RlYm9vayIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKCgojIHB1bGxpbmcgaW4gREZzIGFuZCBNZXJnaW5nIChzdGlsbCBuZWVkZWQ/KQpgYGB7cn0KIyAjcmVkIHdpbmVzCiMgUmVkX3dpbmUgPC0gcmVhZC5jc3YoIndpbmVRdWFsaXR5UmVkcy5jc3YiLCBoZWFkZXI9VFJVRSwgc2VwID0gIiwiKQojIFJlZF93aW5lJFR5cGUgPC0gJ1JlZCcKIyAKIyAjd2hpdGUgd2luZXMKIyBXaGl0ZV93aW5lIDwtIHJlYWQuY3N2KCJ3aW5lUXVhbGl0eVdoaXRlcy5jc3YiLCBoZWFkZXI9VFJVRSwgc2VwID0gIiwiKQojIFdoaXRlX3dpbmUkVHlwZSA8LSAnV2hpdGUnCiMgCiMgIyMgY29uc29saWRhdGVkCiMgRGF0YSA8LSByYmluZChSZWRfd2luZSxXaGl0ZV93aW5lKQojIGRyb3BzIDwtIGMoIlgiKQojIERhdGEgPC0gRGF0YVsgLCAhKG5hbWVzKERhdGEpICVpbiUgZHJvcHMpXQojIERhdGEKIyAKIyAjI2NyZWF0ZSBmaW5hbCBERgojICMgd3JpdGUuY3N2KERhdGEsIi9Vc2Vycy9jb2xpbm9icmllbi9EZXNrdG9wL3JlcG8vc3RhdHNfNjAyMS9TdGF0c19wcm9qZWN0X2dyb3VwXzYvRGF0YS5jc3YiLCByb3cubmFtZXMgPSBGQUxTRSkKIyAjIHdyaXRlLmNzdihEYXRhLCIvVXNlcnMvY29saW5vYnJpZW4vRGVza3RvcC9yZXBvL3N0YXRzXzYwMjEvU3RhdHNfcHJvamVjdF9ncm91cF82L0RhdGEiLCByb3cubmFtZXMgPSBGQUxTRSkKIyAjIyBib3RoIHRoZSBEYXRhIGFuZCBEYXRhIGNzdiBhcmUgdGhlIHNhbWUuIEkga25vdyBwZW9wbGUgcHJlZmVyIG9uZSBmb3JtYXQgdnMgdGhlIG90aGVyIHNvIEkgbWFkZSBib3RoCgpgYGAKCgoKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKIyBsaWJyYXJ5KFJPQ1IpCmxpYnJhcnkoZmFyYXdheSkKbGlicmFyeShkcGx5cikKbGlicmFyeShnZ3Bsb3QyKQpsaWJyYXJ5KHJlc2hhcGUyKQpsaWJyYXJ5KGxlYXBzKQojIGluc3RhbGwucGFja2FnZXMoImJlc3RnbG0iKQpsaWJyYXJ5KGJlc3RnbG0pCiMgaW5zdGFsbC5wYWNrYWdlcygicGVyZm9ybWFuY2UiKQojIGxpYnJhcnkocGVyZm9ybWFuY2UpCmtuaXRyOjpvcHRzX2NodW5rJHNldChlY2hvID0gVFJVRSkKCgoKIyMgTG9hZCBEYXRhc2V0cwpmdWxsX3dpbmVzX2ZpbmFsIDwtIHJlYWQuY3N2KCJEYXRhX0ZpbmFsLmNzdiIsIGhlYWRlciA9IFRSVUUsIHN0cmluZ3NBc0ZhY3RvcnM9VFJVRSkKIyBEcm9wIHF1YWxpdHkgZm9yIHNpbXBsaWNpdHkKZnVsbF93aW5lc19iaW5hcnlfd2l0aF9xdWFsPC1mdWxsX3dpbmVzX2ZpbmFsCmZ1bGxfd2luZXNfYmluYXJ5IDwtIHN1YnNldChmdWxsX3dpbmVzX2ZpbmFsLCBzZWxlY3QgPSAtYyhxdWFsaXR5KSkKIyMgQ29udmVydCB0byAwIGFuZCAxIGZvciByZWFkYWJpbGl0eQpmdWxsX3dpbmVzX2JpbmFyeSRjYXRfcXVhbGl0eSA8LSBhcy5pbnRlZ2VyKGZ1bGxfd2luZXNfYmluYXJ5JGNhdF9xdWFsaXR5ID09ICJIaWdoIikKCnNldC5zZWVkKDkwMjEwKSAjI2ZvciByZXByb2R1Y2liaWxpdHkKc2FtcGxlPC1zYW1wbGUuaW50KG5yb3coZnVsbF93aW5lc19iaW5hcnkpLCBmbG9vciguODAqbnJvdyhmdWxsX3dpbmVzX2JpbmFyeSkpLCByZXBsYWNlID0gRikKdHJhaW48LWZ1bGxfd2luZXNfYmluYXJ5W3NhbXBsZSwgXSAjI3RyYWluaW5nIGRhdGEgZnJhbWUKcm93bmFtZXModHJhaW4pIDwtIGMoMTo1MTk3KQp0ZXN0PC1mdWxsX3dpbmVzX2JpbmFyeVstc2FtcGxlLCBdICMjdGVzdCBkYXRhIGZyYW1lCgojIyBKdXN0IGZvciBhIHNpbmdsZSBib3hwbG90CnRyYWluX3dpdGhfcXVhbDwtZnVsbF93aW5lc19iaW5hcnlfd2l0aF9xdWFsW3NhbXBsZSxdCnRlc3Rfd2l0aF9xdWFsPC1mdWxsX3dpbmVzX2JpbmFyeV93aXRoX3F1YWxbLXNhbXBsZSxdCgoKdHJhaW4KYGBgCiMgRURBCgoKYGBge3J9CiMgZHJvcHNfY2F0cyA8LSBjKCJUeXBlIikKIyBOb19jYXRfdHJhaW4gPC0gdHJhaW5bICwgIShuYW1lcyh0cmFpbikgJWluJSBkcm9wc19jYXRzKV0KIyAjIE5vX1R5cGUKCnBhaXJzKHRyYWluLCBsb3dlci5wYW5lbCA9IE5VTEwpCmBgYAoKYGBge3J9CiMgQ29udmVydCBUeXBlIHRvIGJpbmFyeSB0byAwIGFuZCAxIGZvciBjb3JyZWxhdGlvbgp0cmFpbiRUeXBlIDwtIGFzLmludGVnZXIodHJhaW4kVHlwZSA9PSAiV2hpdGUiKQp0ZXN0JFR5cGUgPC0gYXMuaW50ZWdlcih0ZXN0JFR5cGUgPT0gIldoaXRlIikKY29yX3RyYWluIDwtIGNvcih0cmFpbikKY29yX3RyYWluCmBgYAoKCgpgYGB7cn0KVF9GX2NvciA8LSBhYnMoY29yX3RyYWluKT4uNwpUX0ZfY29yCmBgYApgYGB7cn0KIyMgY3JlYXRlIG1lbHRlZAptZWx0ZWRfY29yX3RyYWluIDwtIG1lbHQoY29yX3RyYWluKQoKIyNjcmVhdGUgaGVhdCBtYXAgQ29uc29saWRhdGVkCmdncGxvdChkYXRhID0gbWVsdGVkX2Nvcl90cmFpbiwgYWVzKHg9VmFyMSwgeT1WYXIyLCBmaWxsPXZhbHVlKSkgKyAKICBnZW9tX3RpbGUoY29sb3IgPSAid2hpdGUiKSsKICBzY2FsZV9maWxsX2dyYWRpZW50Mihsb3cgPSAiYmx1ZSIsIGhpZ2ggPSAicmVkIiwgbWlkID0gIndoaXRlIiwgbWlkcG9pbnQgPSAwLCBsaW1pdCA9IGMoLTEsMSksIHNwYWNlID0gIkxhYiIsIG5hbWU9IlBlYXJzb25cbkNvcnJlbGF0aW9uIikgKwogIHRoZW1lX21pbmltYWwoKSsgCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgdmp1c3QgPSAxLCBzaXplID0gMTIsIGhqdXN0ID0gMSkpKwogIGNvb3JkX2ZpeGVkKCkrCiAgbGFicyh0aXRsZSA9ICdDb25zb2xpZGF0ZWQgKEJvdGggUmVkIGFuZCBXaGl0ZSknKQoKCgoKYGBgCgpgYGB7cn0KIyMgY3JlYXRpbmcgcmVkIGFuZCB3aGl0ZQp0cmFpbl9XaGl0ZSA8LSBmaWx0ZXIodHJhaW4sIFR5cGUgPT0gMSkKdHJhaW5fUmVkIDwtIGZpbHRlcih0cmFpbiwgVHlwZSA9PSAwKQoKCiMjIGRyb3BpbmcgcmVkL3doaXRlIGNvbHVtbnMKdHJhaW5fV2hpdGVfTm9UeXBlIDwtIHN1YnNldCh0cmFpbl9XaGl0ZSwgc2VsZWN0ID0gLWMoVHlwZSkpCnRyYWluX1JlZF9Ob1R5cGUgPC0gc3Vic2V0KHRyYWluX1JlZCwgc2VsZWN0ID0gLWMoVHlwZSkpCgojIyBjcmVhdGluZyBjb3JyZWxhdGlvbnMKY29yX3RyYWluX1doaXRlX05vVHlwZSA8LSBjb3IodHJhaW5fV2hpdGVfTm9UeXBlKQpjb3JfdHJhaW5fUmVkX05vVHlwZSA8LSBjb3IodHJhaW5fUmVkX05vVHlwZSkKCiMjIG1lbHRpbmcKbWVsdGVkX2Nvcl90cmFpbl93aGl0ZSA8LSBtZWx0KGNvcl90cmFpbl9XaGl0ZV9Ob1R5cGUpCm1lbHRlZF9jb3JfdHJhaW5fUmVkIDwtIG1lbHQoY29yX3RyYWluX1JlZF9Ob1R5cGUpCgojI3Bsb3RpbmcKCiMjY3JlYXRlIGhlYXQgbWFwIFdoaXRlCmdncGxvdChkYXRhID0gbWVsdGVkX2Nvcl90cmFpbl93aGl0ZSwgYWVzKHg9VmFyMSwgeT1WYXIyLCBmaWxsPXZhbHVlKSkgKwogIGdlb21fdGlsZShjb2xvciA9ICJ3aGl0ZSIpKwogIHNjYWxlX2ZpbGxfZ3JhZGllbnQyKGxvdyA9ICJibHVlIiwgaGlnaCA9ICJyZWQiLCBtaWQgPSAid2hpdGUiLCBtaWRwb2ludCA9IDAsIGxpbWl0ID0gYygtMSwxKSwgc3BhY2UgPSAiTGFiIiwgbmFtZT0iUGVhcnNvblxuQ29ycmVsYXRpb24iKSArCiAgdGhlbWVfbWluaW1hbCgpKwogIHRoZW1lKGF4aXMudGV4dC54ID0gZWxlbWVudF90ZXh0KGFuZ2xlID0gNDUsIHZqdXN0ID0gMSwgc2l6ZSA9IDEyLCBoanVzdCA9IDEpKSsKICBjb29yZF9maXhlZCgpKwogIGxhYnModGl0bGUgPSAnV2hpdGUgV2luZScpCgojI2NyZWF0ZSBoZWF0IG1hcCBSZWQKZ2dwbG90KGRhdGEgPSBtZWx0ZWRfY29yX3RyYWluX1JlZCwgYWVzKHg9VmFyMSwgeT1WYXIyLCBmaWxsPXZhbHVlKSkgKwogIGdlb21fdGlsZShjb2xvciA9ICJ3aGl0ZSIpKwogIHNjYWxlX2ZpbGxfZ3JhZGllbnQyKGxvdyA9ICJibHVlIiwgaGlnaCA9ICJyZWQiLCBtaWQgPSAid2hpdGUiLCBtaWRwb2ludCA9IDAsIGxpbWl0ID0gYygtMSwxKSwgc3BhY2UgPSAiTGFiIiwgbmFtZT0iUGVhcnNvblxuQ29ycmVsYXRpb24iKSArCiAgdGhlbWVfbWluaW1hbCgpKwogIHRoZW1lKGF4aXMudGV4dC54ID0gZWxlbWVudF90ZXh0KGFuZ2xlID0gNDUsIHZqdXN0ID0gMSwgc2l6ZSA9IDEyLCBoanVzdCA9IDEpKSsKICBjb29yZF9maXhlZCgpKwogIGxhYnModGl0bGUgPSAnUmVkIFdpbmUnKQoKYGBgCgpgYGB7cn0KZ2dwbG90KGRhdGEgPSB0cmFpbiwgbWFwcGluZyA9IGFlcyh4PVR5cGUpKSArIAogIGdlb21fYmFyKCkKYGBgCgojIFJlZ3Jlc3Npb24gVGVzdGluZwoKYGBge3J9CiMjIHByZXNzIGZvcm11bGEgKGZyb20gY2xhc3MpCmdldF9wcmVzcyA8LSBmdW5jdGlvbihtb2RlbCkgewogIHN1bSgoKG1vZGVsJHJlc2lkdWFscykvICgxLSAobG0uaW5mbHVlbmNlKG1vZGVsKSRoYXQpKSleMikKfQpgYGAKCgpgYGB7cn0KIyMgZmlyc3QgZ28KZnVsbDwtZ2xtKGNhdF9xdWFsaXR5fi4sIGZhbWlseT1iaW5vbWlhbCwgZGF0YT10cmFpbikKc3VtbWFyeShmdWxsKQpgYGAKYGBge3J9CiMjIHJlbW92ZWQgYWxsIGluc2lnbmlmaWNhbnQKcmVkdWNlZF8xPC1nbG0oZm9ybXVsYSA9IGNhdF9xdWFsaXR5fnZvbGF0aWxlLmFjaWRpdHkrcmVzaWR1YWwuc3VnYXIrZnJlZS5zdWxmdXIuZGlveGlkZSt0b3RhbC5zdWxmdXIuZGlveGlkZStkZW5zaXR5K3BIK3N1bHBoYXRlcythbGNvaG9sK1R5cGUsIGZhbWlseT1iaW5vbWlhbCwgZGF0YT10cmFpbikKc3VtbWFyeShyZWR1Y2VkXzEpCmBgYAoKYGBge3J9CiMjZXZhbHVhdGluZyBtb2RlbApSZWR1Y2VkMV9BSUNfdHJhaW4gPC0gcmVkdWNlZF8xJGFpYwoKIyNwcmVkaWN0ZWQgcXVhbGl0eSBmb3IgdGVzdCBkYXRhIGJhc2VkIG9uIHRyYWluaW5nIGRhdGEKcHJlZHM8LXByZWRpY3QocmVkdWNlZF8xLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQoKcmVkdWNlZF8xX2Vycm9yIDwtIHRhYmxlKHRlc3QkY2F0X3F1YWxpdHksIHByZWRzPjAuNSkKCnJlZHVjZWRfMV9lcnJvcgoKZXZ1bGF0aW9uX3N1bW1hcnkgPC0gZGF0YS5mcmFtZSgKICBhdHRlbXB0ID0gJ3JlZHVjZWRfMScsCiAgQUlDID0gUmVkdWNlZDFfQUlDX3RyYWluLAogIFBSRVNTID0gZ2V0X3ByZXNzKHJlZHVjZWRfMSksCiAgJ0ZhbHNlIHBvc2l0aXZlJyA9IHJvdW5kKHJlZHVjZWRfMV9lcnJvclszXS8ocmVkdWNlZF8xX2Vycm9yWzFdK3JlZHVjZWRfMV9lcnJvclszXSksMyksCiAgJ0ZhbHNlIG5lZ2F0aXZlJyA9IHJvdW5kKHJlZHVjZWRfMV9lcnJvclsyXS8ocmVkdWNlZF8xX2Vycm9yWzJdK3JlZHVjZWRfMV9lcnJvcls0XSksMyksCiAgJ0Vycm9yIFJhdGUnID0gcm91bmQoKHJlZHVjZWRfMV9lcnJvclsyXStyZWR1Y2VkXzFfZXJyb3JbM10pLyhyZWR1Y2VkXzFfZXJyb3JbMV0rcmVkdWNlZF8xX2Vycm9yWzJdK3JlZHVjZWRfMV9lcnJvclszXStyZWR1Y2VkXzFfZXJyb3JbNF0pLDMpCikKZXZ1bGF0aW9uX3N1bW1hcnkKYGBgCgojIyBzZWNvbmQgbW9kZWwKIyMgaHR0cHM6Ly9yc3R1ZGlvLXB1YnMtc3RhdGljLnMzLmFtYXpvbmF3cy5jb20vMjg5N185MjIwYjIxY2ZjMGM0M2EzOTZmZjlhYmYxMjJiYjM1MS5odG1sCgpgYGB7cn0KIyBpbnN0YWxsLnBhY2thZ2VzKCJiZXN0Z2xtIikKIyMgUHJlcGFyZSBkYXRhCnRyYWluLmZvci5iZXN0LmxvZ2lzdGljIDwtIHdpdGhpbih0cmFpbiwgewogICAgeSA8LSBjYXRfcXVhbGl0eSAKfSkKCiMjIFJlb3JkZXIgdmFyaWFibGVzCnRyYWluLmZvci5iZXN0LmxvZ2lzdGljIDwtCiAgICB0cmFpbi5mb3IuYmVzdC5sb2dpc3RpY1ssIGMoImZpeGVkLmFjaWRpdHkiLCJ2b2xhdGlsZS5hY2lkaXR5IiwiY2l0cmljLmFjaWQiLCJyZXNpZHVhbC5zdWdhciIsInRvdGFsLnN1bGZ1ci5kaW94aWRlIiwiZGVuc2l0eSIsImNobG9yaWRlcyIsImZyZWUuc3VsZnVyLmRpb3hpZGUiLCdwSCcsJ3N1bHBoYXRlcycsJ2FsY29ob2wnLCdUeXBlJywieSIpXQoKIyMgUGVyZm9ybQpyZXMuYmVzdC5sb2dpc3RpYyA8LQogICAgYmVzdGdsbShYeSA9IHRyYWluLmZvci5iZXN0LmxvZ2lzdGljLAogICAgICAgICAgICBmYW1pbHkgPSBiaW5vbWlhbCwgICAgICAgICAgIyBiaW5vbWlhbCBmYW1pbHkgZm9yIGxvZ2lzdGljCiAgICAgICAgICAgIElDID0gIkFJQyIsICAgICAgICAgICAgICAgICAjIEluZm9ybWF0aW9uIGNyaXRlcmlhIGZvcgogICAgICAgICAgICBtZXRob2QgPSAiZXhoYXVzdGl2ZSIpCmBgYAoKCmBgYHtyfQpyZXMuYmVzdC5sb2dpc3RpYyRCZXN0TW9kZWxzCnN1bW1hcnkocmVzLmJlc3QubG9naXN0aWMkQmVzdE1vZGVsKQpgYGAKYGBge3J9CnJlZHVjZWRfNCA8LSByZXMuYmVzdC5sb2dpc3RpYyRCZXN0TW9kZWwKIyNldmFsdWF0aW5nIG1vZGVsClJlZHVjZWQ0X0FJQ190cmFpbiA8LSByZWR1Y2VkXzQkYWljCgojI3ByZWRpY3RlZCBxdWFsaXR5IGZvciB0ZXN0IGRhdGEgYmFzZWQgb24gdHJhaW5pbmcgZGF0YQpwcmVkczwtcHJlZGljdChyZWR1Y2VkXzQsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCgpyZWR1Y2VkXzRfZXJyb3IgPC0gdGFibGUodGVzdCRjYXRfcXVhbGl0eSwgcHJlZHM+MC41KQoKZXZ1bGF0aW9uX3N1bW1hcnlfNCA8LSBkYXRhLmZyYW1lKAogIGF0dGVtcHQgPSAncmVkdWNlZF80X2Vycm9yIChhbGwgcG9zc2libGUpJywKICBBSUMgPSBSZWR1Y2VkNF9BSUNfdHJhaW4sCiAgUFJFU1MgPSBnZXRfcHJlc3MocmVkdWNlZF80KSwKICAnRmFsc2UgcG9zaXRpdmUnID0gcm91bmQocmVkdWNlZF80X2Vycm9yWzNdLyhyZWR1Y2VkXzRfZXJyb3JbMV0rcmVkdWNlZF80X2Vycm9yWzNdKSwzKSwKICAnRmFsc2UgbmVnYXRpdmUnID0gcm91bmQocmVkdWNlZF80X2Vycm9yWzJdLyhyZWR1Y2VkXzRfZXJyb3JbMl0rcmVkdWNlZF80X2Vycm9yWzRdKSwzKSwKICAnRXJyb3IgUmF0ZScgPSByb3VuZCgocmVkdWNlZF80X2Vycm9yWzJdK3JlZHVjZWRfNF9lcnJvclszXSkvKHJlZHVjZWRfNF9lcnJvclsxXStyZWR1Y2VkXzRfZXJyb3JbMl0rcmVkdWNlZF80X2Vycm9yWzNdK3JlZHVjZWRfNF9lcnJvcls0XSksMykKKQoKZXZ1bGF0aW9uX3N1bW1hcnkgPC0gcmJpbmQoZXZ1bGF0aW9uX3N1bW1hcnksZXZ1bGF0aW9uX3N1bW1hcnlfNCkKIyBldnVsYXRpb25fc3VtbWFyeQpgYGAKCmBgYHtyfQojIGRhdGEuZnJhbWUoY2hlY2tfY29sbGluZWFyaXR5KHJlZHVjZWRfNCkpCgoKCgoKCgoKCgojY29tZSBiYWNrIGFuZCBhZGQgZGYgc3R1ZmYKYGBgCgojIyBpbiBhbiBlZmZvcnQgdG8gbG93ZXIgVklGcyBzY29yZXMgYW5kIGNvcnJlbGF0aW9uLCBJIGFtIHJlbW92aW5nIGZpeGVkLmFjaWRpdHkKCmBgYHtyfQpyZWR1Y2VkXzRfMjwtZ2xtKGNhdF9xdWFsaXR5fnZvbGF0aWxlLmFjaWRpdHkrY2l0cmljLmFjaWQrcmVzaWR1YWwuc3VnYXIrdG90YWwuc3VsZnVyLmRpb3hpZGUrZGVuc2l0eStmcmVlLnN1bGZ1ci5kaW94aWRlK3BIK3N1bHBoYXRlcythbGNvaG9sK1R5cGUsIGZhbWlseT1iaW5vbWlhbCwgZGF0YT10cmFpbikKc3VtbWFyeShyZWR1Y2VkXzRfMikKYGBgCmBgYHtyfQojI2V2YWx1YXRpbmcgbW9kZWwKUmVkdWNlZDRfMl9BSUNfdHJhaW4gPC0gcmVkdWNlZF80XzIkYWljCiMjcHJlZGljdGVkIHF1YWxpdHkgZm9yIHRlc3QgZGF0YSBiYXNlZCBvbiB0cmFpbmluZyBkYXRhCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF8yLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyZWR1Y2VkXzRfMl9lcnJvciA8LSB0YWJsZSh0ZXN0JGNhdF9xdWFsaXR5LCBwcmVkcz4wLjcpCiNDdXJ2ZXMKZXZ1bGF0aW9uX3N1bW1hcnlfNF8yIDwtIGRhdGEuZnJhbWUoCiAgYXR0ZW1wdCA9ICdyZWR1Y2VkXzRfMl9lcnJvciAocG9zdCBWSUYgYWRqdXN0bWVudHMpJywKICBBSUMgPSBSZWR1Y2VkNF8yX0FJQ190cmFpbiwKICBQUkVTUyA9IGdldF9wcmVzcyhyZWR1Y2VkXzRfMiksCiAgJ0ZhbHNlIHBvc2l0aXZlJyA9IHJvdW5kKHJlZHVjZWRfNF8yX2Vycm9yWzNdLyhyZWR1Y2VkXzRfMl9lcnJvclsxXStyZWR1Y2VkXzRfMl9lcnJvclszXSksMyksCiAgJ0ZhbHNlIG5lZ2F0aXZlJyA9IHJvdW5kKHJlZHVjZWRfNF8yX2Vycm9yWzJdLyhyZWR1Y2VkXzRfMl9lcnJvclsyXStyZWR1Y2VkXzRfMl9lcnJvcls0XSksMyksCiAgJ0Vycm9yIFJhdGUnID0gcm91bmQoKHJlZHVjZWRfNF8yX2Vycm9yWzJdK3JlZHVjZWRfNF8yX2Vycm9yWzNdKS8ocmVkdWNlZF80XzJfZXJyb3JbMV0rcmVkdWNlZF80XzJfZXJyb3JbMl0rcmVkdWNlZF80XzJfZXJyb3JbM10rcmVkdWNlZF80XzJfZXJyb3JbNF0pLDMpCikKCmV2dWxhdGlvbl9zdW1tYXJ5IDwtIHJiaW5kKGV2dWxhdGlvbl9zdW1tYXJ5LGV2dWxhdGlvbl9zdW1tYXJ5XzRfMikKZXZ1bGF0aW9uX3N1bW1hcnkKCgpgYGAKCiMjIG5vdyBsb29raW5nIGF0IG91dGxpZXJzICh3aXRoICJiZXN0IHBvc3NpYmxlIikKCmBgYHtyfQpzdW1tYXJ5KHJlZHVjZWRfNCkKCmBgYAoKCiMgTm93IGxvb2tpbmcgaW50byBvdXRsaWVycy9pbmZsdWVuY2UKCmBgYHtyfQpwIDwtIDEyCm4gPC0gNTE5NwpgYGAKCiMjIyBDb29rcwpgYGB7cn0KcmVkdWNlZF80X2Nvb2sgPC1jb29rcy5kaXN0YW5jZShyZWR1Y2VkXzQpCnJlZHVjZWRfNF9jb29rW3JlZHVjZWRfNF9jb29rPnFmKDAuNSxwLG4tcCldCmBgYAojIyMgREZGSVRzCgpgYGB7cn0KIyNkZmZpdHMKREZGSVRTPC1kZmZpdHMocmVkdWNlZF80KQpEREZGSVRTX2luZmx1ZW5jZSA8LSBERkZJVFNbYWJzKERGRklUUyk+MipzcXJ0KHAvbildCkRERkZJVFNfaW5mbHVlbmNlCmBgYAojIyMgREZCRVRBcwpgYGB7cn0KREZCRVRBUzwtZGZiZXRhcyhyZWR1Y2VkXzQpCmFicyhERkJFVEFTKT4yL3NxcnQobikKYGBgCgoKIyMjIGxldmVyYWdlCmBgYHtyfQojI2xldmVyYWdlcwpsZXY8LWxtLmluZmx1ZW5jZShyZWR1Y2VkXzQpJGhhdAojI2lkZW50aWZ5IGhpZ2ggbGV2ZXJhZ2UgcG9pbnRzCmxldmVyYWdlcyA8LSBsZXZbbGV2PjIqcC9uXQpsZXZlcmFnZXMKYGBgCgoKIyMjIG91dGxpZXIKCmBgYHtyfQpyZWR1Y2VkXzQucmVzIDwtIHJlZHVjZWRfNCRyZXNpZHVhbHMKY3JpdDwtcXQoMS0wLjA1LygyKm4pLCBuLXAtMSkKb3V0bGllcnMgPC0gcmVkdWNlZF80LnJlc1thYnMocmVkdWNlZF80LnJlcyk+Y3JpdF0Kb3V0bGllcnMKYGBgCgoKCgpgYGB7cn0KIyMgb3V0bGllcnMgcmVtb3ZlZApvdXRsaWVyc19pbmRleCA8LSBhdHRyKG91dGxpZXJzLCAibmFtZXMiKQpvdXRsaWVyc19pbmRleCA8LSBhcy5udW1lcmljKG91dGxpZXJzX2luZGV4KQp0cmFpbl9ub19vdXRsaWVycyA8LSB0cmFpblstKG91dGxpZXJzX2luZGV4KSxdCgojbGV2ZXJhZ2VzIHJlbW92ZWQKbGVyZXJhZ2VzX2luZGV4IDwtIGF0dHIobGV2ZXJhZ2VzLCAibmFtZXMiKQpsZXJlcmFnZXNfaW5kZXggPC0gYXMubnVtZXJpYyhsZXJlcmFnZXNfaW5kZXgpCnRyYWluX25vX2xldmVyYWdlcyA8LSB0cmFpblstKGxlcmVyYWdlc19pbmRleCksXQoKIyBEREZGSVRTX2luZmx1ZW5jZQpEREZGSVRTX2luZGV4IDwtIGF0dHIoRERGRklUU19pbmZsdWVuY2UsICJuYW1lcyIpCkRERkZJVFNfaW5kZXggPC0gYXMubnVtZXJpYyhEREZGSVRTX2luZGV4KQp0cmFpbl9ub19EREZGSVRTIDwtIHRyYWluWy0oRERGRklUU19pbmRleCksXQoKIyBhbGwgIm5vbi1ub3JtYWwiIHJlbW92ZWQKYWxsX3NwZWNpYWwgPC0gYyhEREZGSVRTX2luZGV4LGxlcmVyYWdlc19pbmRleCxvdXRsaWVyc19pbmRleCkKdHJhaW5fbm90aGluZ19zcGVjaWFsIDwtIHRyYWluWy0oYWxsX3NwZWNpYWwpLF0KdHJhaW5fbm90aGluZ19zcGVjaWFsCgoKCgpgYGAKYGBge3J9CnZpZih0cmFpbltjKDIsMyw0LDcsOCw2LDksMTAsMTEpXSkKYGBgCgpgYGB7cn0KdHJhaW5fdGVtcDwtdHJhaW4KIyBhcy5mYWN0b3IodHJhaW5fdGVtcCRUeXBlKTwtbnVtZXJpYyh0cmFpbl90ZW1wJFR5cGUpCiN0cmFpbl90ZW1wCiMgYXMuZmFjdG9yCgp0cmFpbl90ZW1wJFR5cGUgPC0gYXMubnVtZXJpYyh0cmFpbl90ZW1wJFR5cGUpLTEKdHJhaW5fdGVtcCRUeXBlIDwtIGFzLmludGVnZXIodHJhaW5fdGVtcCRUeXBlKQp0cmFpbl90ZW1wCmBgYAoKCgojIyBjcmVhdGluZyByZWR1Y2VkIAoKYGBge3J9CnJlZHVjZWRfNF8zIDwtIGdsbShjYXRfcXVhbGl0eX52b2xhdGlsZS5hY2lkaXR5K2NpdHJpYy5hY2lkK3Jlc2lkdWFsLnN1Z2FyK3RvdGFsLnN1bGZ1ci5kaW94aWRlK2RlbnNpdHkrZnJlZS5zdWxmdXIuZGlveGlkZStwSCtzdWxwaGF0ZXMrYWxjb2hvbCtUeXBlLCBmYW1pbHk9Ymlub21pYWwsIGRhdGE9dHJhaW5fbm9fb3V0bGllcnMpCgpyZWR1Y2VkXzRfNF9sZXYgPC0gZ2xtKGNhdF9xdWFsaXR5fnZvbGF0aWxlLmFjaWRpdHkrY2l0cmljLmFjaWQrcmVzaWR1YWwuc3VnYXIrdG90YWwuc3VsZnVyLmRpb3hpZGUrZGVuc2l0eStmcmVlLnN1bGZ1ci5kaW94aWRlK3BIK3N1bHBoYXRlcythbGNvaG9sK1R5cGUsIGZhbWlseT1iaW5vbWlhbCwgZGF0YT10cmFpbl9ub19sZXZlcmFnZXMpCgpyZWR1Y2VkXzRfNV9EREZGSVRTIDwtIGdsbShjYXRfcXVhbGl0eX52b2xhdGlsZS5hY2lkaXR5K2NpdHJpYy5hY2lkK3Jlc2lkdWFsLnN1Z2FyK3RvdGFsLnN1bGZ1ci5kaW94aWRlK2RlbnNpdHkrZnJlZS5zdWxmdXIuZGlveGlkZStwSCtzdWxwaGF0ZXMrYWxjb2hvbCtUeXBlLCBmYW1pbHk9Ymlub21pYWwsIGRhdGE9dHJhaW5fbm9fRERGRklUUykKCnJlZHVjZWRfNF82X25vX3NwZWNpYWwgPC0gZ2xtKGNhdF9xdWFsaXR5fnZvbGF0aWxlLmFjaWRpdHkrY2l0cmljLmFjaWQrcmVzaWR1YWwuc3VnYXIrdG90YWwuc3VsZnVyLmRpb3hpZGUrZGVuc2l0eStmcmVlLnN1bGZ1ci5kaW94aWRlK3BIK3N1bHBoYXRlcythbGNvaG9sK1R5cGUsIGZhbWlseT1iaW5vbWlhbCwgZGF0YT10cmFpbl9ub3RoaW5nX3NwZWNpYWwpCiMgc3VtbWFyeShyZWR1Y2VkXzRfNl9ub19zcGVjaWFsKQoKCgpgYGAKCgpgYGB7cn0KIyMgY2hlY2tpbmcgY29saW5lYXJpdHkgLyBWSUYgc2NvcmVzCiMgcmVkdWNlZF80XzNfY29sIDwtIGRhdGEuZnJhbWUoJ3JlZHVjZWRfNF8zJyA9IGNoZWNrX2NvbGxpbmVhcml0eShyZWR1Y2VkXzRfMykpCiMgcmVkdWNlZF80XzNfY29sX1ZJRiA8LSByZWR1Y2VkXzRfM19jb2xbYygncmVkdWNlZF80XzMuVGVybScsJ3JlZHVjZWRfNF8zLlZJRicpXQojIHJlZHVjZWRfNF8zX2NvbF9WSUYKIyAKIyByZWR1Y2VkXzRfNF9sZXZfY29sIDwtIGRhdGEuZnJhbWUoJ3JlZHVjZWRfNF80X2xldicgPSBjaGVja19jb2xsaW5lYXJpdHkocmVkdWNlZF80XzRfbGV2KSkKIyByZWR1Y2VkXzRfNF9sZXZfY29sX1ZJRiA8LSByZWR1Y2VkXzRfNF9sZXZfY29sW2MoJ3JlZHVjZWRfNF80X2xldi5UZXJtJywncmVkdWNlZF80XzRfbGV2LlZJRicpXQojIHJlZHVjZWRfNF80X2xldl9jb2xfVklGCiMgCiMgCiMgcmVkdWNlZF80XzVfRERGRklUU19jb2wgPC0gZGF0YS5mcmFtZSgncmVkdWNlZF80XzVfRERGRklUUycgPSBjaGVja19jb2xsaW5lYXJpdHkocmVkdWNlZF80XzVfRERGRklUUykpCiMgcmVkdWNlZF80XzVfRERGRklUU19jb2xfVklGIDwtIHJlZHVjZWRfNF81X0RERkZJVFNfY29sW2MoJ3JlZHVjZWRfNF81X0RERkZJVFMuVGVybScsJ3JlZHVjZWRfNF81X0RERkZJVFMuVklGJyldCiMgcmVkdWNlZF80XzVfRERGRklUU19jb2xfVklGCiMgCiMgcmVkdWNlZF80XzZfbm9fc3BlY2lhbF9jb2wgPC0gZGF0YS5mcmFtZSgncmVkdWNlZF80XzZfbm9fc3BlY2lhbCcgPSBjaGVja19jb2xsaW5lYXJpdHkocmVkdWNlZF80XzZfbm9fc3BlY2lhbCkpCiMgcmVkdWNlZF80XzZfbm9fc3BlY2lhbF9jb2xfVklGIDwtIHJlZHVjZWRfNF82X25vX3NwZWNpYWxfY29sW2MoJ3JlZHVjZWRfNF82X25vX3NwZWNpYWwuVGVybScsJ3JlZHVjZWRfNF82X25vX3NwZWNpYWwuVklGJyldCiMgcmVkdWNlZF80XzZfbm9fc3BlY2lhbF9jb2xfVklGCiMgCiMgVklGX3N1bW1hcnkgPC0gZGF0YS5mcmFtZSgnMCc9cmVkdWNlZF80XzNfY29sX1ZJRlsncmVkdWNlZF80XzMuVGVybSddLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgJzEnPXJlZHVjZWRfNF8zX2NvbF9WSUZbJ3JlZHVjZWRfNF8zLlZJRiddLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgJzInPXJlZHVjZWRfNF80X2xldl9jb2xfVklGWydyZWR1Y2VkXzRfNF9sZXYuVklGJ10sCiMgICAgICAgICAgICAgICAgICAgICAgICAgICAnMyc9cmVkdWNlZF80XzVfRERGRklUU19jb2xfVklGWydyZWR1Y2VkXzRfNV9EREZGSVRTLlZJRiddLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgJzQnPXJlZHVjZWRfNF82X25vX3NwZWNpYWxfY29sX1ZJRlsncmVkdWNlZF80XzZfbm9fc3BlY2lhbC5WSUYnXSkKIyBjb2xuYW1lcyhWSUZfc3VtbWFyeSkgPC0gYygnUHJlZGljdG9yIFZhcmlhYmxlJywnNF8zLlZJRi5PdXRsaWVycycsJzRfNF9sZXYuVklGJywnNF81X0RERkZJVFMuVklGJywnNF82X25vX3NwZWNpYWwuVklGJykKIyBWSUZfc3VtbWFyeQoKIyMgVklGIGZvciBPdXRsaWVycwojIyMgY2F0X3F1YWxpdHl+dm9sYXRpbGUuYWNpZGl0eStjaXRyaWMuYWNpZCtyZXNpZHVhbC5zdWdhcit0b3RhbC5zdWxmdXIuZGlveGlkZStkZW5zaXR5K2ZyZWUuc3VsZnVyLmRpb3hpZGUrcEgrc3VscGhhdGVzK2FsY29ob2wrVHlwZQoKCiMKcmVnXzRfVklGX3Rlc3QgPC0gdmlmKHRyYWluX3RlbXBbYygxLDIsMyw0LDcsOCw2LDksMTAsMTEsMTIpXSkKcmVnXzRfMl9WSUZfdGVzdCA8LSB2aWYodHJhaW5fdGVtcFtjKDIsMyw0LDcsOCw2LDksMTAsMTEsMTIpXSkKb3V0bGllcnNfVklGIDwtIHZpZih0cmFpbl9ub19vdXRsaWVyc1tjKDIsMyw0LDcsOCw2LDksMTAsMTEsMTIpXSkKbGV2ZXJhZ2VfVklGIDwtIHZpZih0cmFpbl9ub19sZXZlcmFnZXNbYygyLDMsNCw3LDgsNiw5LDEwLDExLDEyKV0pCkRERkZJVFNfVklGIDwtIHZpZih0cmFpbl9ub19EREZGSVRTW2MoMiwzLDQsNyw4LDYsOSwxMCwxMSwxMildKQpub3RoaW5nX3NwZWNpYWwgPC0gdmlmKHRyYWluX25vdGhpbmdfc3BlY2lhbFtjKDIsMyw0LDcsOCw2LDksMTAsMTEsMTIpXSkKCgpyZWdfNF9WSUZfdGVzdAoKVklGX3N1bW1hcnlfdGVzdCA8LSBkYXRhLmZyYW1lKCdiZXN0X3Bvc3NpYmxlX1ZJRiAocG9zdCknPXJlZ180XzJfVklGX3Rlc3QsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnb3V0bGllcnNfVklGJz1vdXRsaWVyc19WSUYsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbGV2ZXJhZ2VfVklGJz1sZXZlcmFnZV9WSUYsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnRERGRklUU19WSUYnPSBEREZGSVRTX1ZJRiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdub3RoaW5nX3NwZWNpYWwnPW5vdGhpbmdfc3BlY2lhbCkKVklGX3N1bW1hcnlfdGVzdAoKYGBgCgoKYGBge3J9CiMjZXZhbHVhdGluZyBtb2RlbApSZWR1Y2VkNF8zX0FJQ190cmFpbiA8LSByZWR1Y2VkXzRfMyRhaWMKCiMjcHJlZGljdGVkIHF1YWxpdHkgZm9yIHRlc3QgZGF0YSBiYXNlZCBvbiB0cmFpbmluZyBkYXRhCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF8zLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQoKcmVkdWNlZF80XzNfZXJyb3IgPC0gdGFibGUodGVzdCRjYXRfcXVhbGl0eSwgcHJlZHM+MC42KQoKZXZ1bGF0aW9uX3N1bW1hcnlfNF8zIDwtIGRhdGEuZnJhbWUoCiAgYXR0ZW1wdCA9ICdyZWR1Y2VkXzRfM19lcnJvcl9vdXRsaWVycycsCiAgQUlDID0gUmVkdWNlZDRfM19BSUNfdHJhaW4sCiAgUFJFU1MgPSBnZXRfcHJlc3MocmVkdWNlZF80XzMpLAogICdGYWxzZSBwb3NpdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfM19lcnJvclszXS8ocmVkdWNlZF80XzNfZXJyb3JbMV0rcmVkdWNlZF80XzNfZXJyb3JbM10pLDMpLAogICdGYWxzZSBuZWdhdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfM19lcnJvclsyXS8ocmVkdWNlZF80XzNfZXJyb3JbMl0rcmVkdWNlZF80XzNfZXJyb3JbNF0pLDMpLAogICdFcnJvciBSYXRlJyA9IHJvdW5kKChyZWR1Y2VkXzRfM19lcnJvclsyXStyZWR1Y2VkXzRfM19lcnJvclszXSkvKHJlZHVjZWRfNF8zX2Vycm9yWzFdK3JlZHVjZWRfNF8zX2Vycm9yWzJdK3JlZHVjZWRfNF8zX2Vycm9yWzNdK3JlZHVjZWRfNF8zX2Vycm9yWzRdKSwzKQopCgpldnVsYXRpb25fc3VtbWFyeSA8LSByYmluZChldnVsYXRpb25fc3VtbWFyeSxldnVsYXRpb25fc3VtbWFyeV80XzMpCmV2dWxhdGlvbl9zdW1tYXJ5CmBgYAoKCmBgYHtyfQojI2V2YWx1YXRpbmcgbW9kZWwgbGV2ZXJhZ2UKcmVkdWNlZF80XzRfbGV2X0FJQ190cmFpbiA8LSByZWR1Y2VkXzRfNF9sZXYkYWljCgojI3ByZWRpY3RlZCBxdWFsaXR5IGZvciB0ZXN0IGRhdGEgYmFzZWQgb24gdHJhaW5pbmcgZGF0YQpwcmVkczwtcHJlZGljdChyZWR1Y2VkXzRfNF9sZXYsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCgpyZWR1Y2VkXzRfNF9sZXZfZXJyb3IgPC0gdGFibGUodGVzdCRjYXRfcXVhbGl0eSwgcHJlZHM+MC42NSkKCmV2dWxhdGlvbl9zdW1tYXJ5XzRfNF9sZXYgPC0gZGF0YS5mcmFtZSgKICBhdHRlbXB0ID0gJ3JlZHVjZWRfNF80X2xldl9lcnJvcicsCiAgQUlDID0gcmVkdWNlZF80XzRfbGV2X0FJQ190cmFpbiwKICBQUkVTUyA9IGdldF9wcmVzcyhyZWR1Y2VkXzRfNF9sZXYpLAogICdGYWxzZSBwb3NpdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbM10vKHJlZHVjZWRfNF80X2xldl9lcnJvclsxXStyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbM10pLDMpLAogICdGYWxzZSBuZWdhdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbMl0vKHJlZHVjZWRfNF80X2xldl9lcnJvclsyXStyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbNF0pLDMpLAogICdFcnJvciBSYXRlJyA9IHJvdW5kKChyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbMl0rcmVkdWNlZF80XzRfbGV2X2Vycm9yWzNdKS8ocmVkdWNlZF80XzRfbGV2X2Vycm9yWzFdK3JlZHVjZWRfNF80X2xldl9lcnJvclsyXStyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbM10rcmVkdWNlZF80XzRfbGV2X2Vycm9yWzRdKSwzKQopCgpldnVsYXRpb25fc3VtbWFyeSA8LSByYmluZChldnVsYXRpb25fc3VtbWFyeSxldnVsYXRpb25fc3VtbWFyeV80XzRfbGV2KQpldnVsYXRpb25fc3VtbWFyeQpgYGAKCgoKYGBge3J9CiMjZXZhbHVhdGluZyBtb2RlbCBEREZGSVRTCnJlZHVjZWRfNF81X0RERkZJVFNfQUlDX3RyYWluIDwtIHJlZHVjZWRfNF81X0RERkZJVFMkYWljCgojI3ByZWRpY3RlZCBxdWFsaXR5IGZvciB0ZXN0IGRhdGEgYmFzZWQgb24gdHJhaW5pbmcgZGF0YQpwcmVkczwtcHJlZGljdChyZWR1Y2VkXzRfNV9EREZGSVRTLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQoKcmVkdWNlZF80XzVfRERGRklUU19lcnJvciA8LSB0YWJsZSh0ZXN0JGNhdF9xdWFsaXR5LCBwcmVkcz4wLjcpCgpldnVsYXRpb25fc3VtbWFyeV80XzVfRERGRklUUyA8LSBkYXRhLmZyYW1lKAogIGF0dGVtcHQgPSAncmVkdWNlZF80XzVfRERGRklUU19lcnJvcicsCiAgQUlDID0gcmVkdWNlZF80XzVfRERGRklUU19BSUNfdHJhaW4sCiAgUFJFU1MgPSBnZXRfcHJlc3MocmVkdWNlZF80XzVfRERGRklUUyksCiAgJ0ZhbHNlIHBvc2l0aXZlJyA9IHJvdW5kKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbM10vKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbMV0rcmVkdWNlZF80XzVfRERGRklUU19lcnJvclszXSksMyksCiAgJ0ZhbHNlIG5lZ2F0aXZlJyA9IHJvdW5kKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbMl0vKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbMl0rcmVkdWNlZF80XzVfRERGRklUU19lcnJvcls0XSksMyksCiAgJ0Vycm9yIFJhdGUnID0gcm91bmQoKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbMl0rcmVkdWNlZF80XzVfRERGRklUU19lcnJvclszXSkvKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbMV0rcmVkdWNlZF80XzVfRERGRklUU19lcnJvclsyXStyZWR1Y2VkXzRfNV9EREZGSVRTX2Vycm9yWzNdK3JlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbNF0pLDMpCikKCmV2dWxhdGlvbl9zdW1tYXJ5IDwtIHJiaW5kKGV2dWxhdGlvbl9zdW1tYXJ5LGV2dWxhdGlvbl9zdW1tYXJ5XzRfNV9EREZGSVRTKQpldnVsYXRpb25fc3VtbWFyeQpgYGAKCmBgYHtyfQojI2V2YWx1YXRpbmcgbW9kZWwgRERGRklUUwpyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX0FJQ190cmFpbiA8LSByZWR1Y2VkXzRfNl9ub19zcGVjaWFsJGFpYwoKIyNwcmVkaWN0ZWQgcXVhbGl0eSBmb3IgdGVzdCBkYXRhIGJhc2VkIG9uIHRyYWluaW5nIGRhdGEKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzZfbm9fc3BlY2lhbCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKCnJlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3IgPC0gdGFibGUodGVzdCRjYXRfcXVhbGl0eSwgcHJlZHM+MC44KQoKZXZ1bGF0aW9uX3N1bW1hcnlfNF82X25vX3NwZWNpYWwgPC0gZGF0YS5mcmFtZSgKICBhdHRlbXB0ID0gJ3JlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3InLAogIEFJQyA9IHJlZHVjZWRfNF82X25vX3NwZWNpYWxfQUlDX3RyYWluLAogIFBSRVNTID0gZ2V0X3ByZXNzKHJlZHVjZWRfNF82X25vX3NwZWNpYWwpLAogICdGYWxzZSBwb3NpdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzNdLyhyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzFdK3JlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3JbM10pLDMpLAogICdGYWxzZSBuZWdhdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzJdLyhyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzJdK3JlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3JbNF0pLDMpLAogICdFcnJvciBSYXRlJyA9IHJvdW5kKChyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzJdK3JlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3JbM10pLyhyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzFdK3JlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3JbMl0rcmVkdWNlZF80XzZfbm9fc3BlY2lhbF9lcnJvclszXStyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzRdKSwzKQopCgpldnVsYXRpb25fc3VtbWFyeSA8LSByYmluZChldnVsYXRpb25fc3VtbWFyeSxldnVsYXRpb25fc3VtbWFyeV80XzZfbm9fc3BlY2lhbCkKZXZ1bGF0aW9uX3N1bW1hcnkKYGBgCgoKIyMgUk9DIEN1cnZlcyBhbmQgQVVDCmBgYHtyfQojIyByZWR1Y2VkXzEKIyBkZXRhY2gocGFja2FnZTpwZXJmb3JtYW5jZSwgdW5sb2FkPVRSVUUpCiMjIEZZSSB0aGUgcGVyZm9ybWFuY2UgcGFja2FnZSBjYXVzZXMgUk9DIGN1cnZlcyB0byBub3Qgd29yawpsaWJyYXJ5KFJPQ1IpCgoKCiMgcmVkdWNlZF8xCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfMSxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfMSIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzwtcGVyZm9ybWFuY2UocmF0ZXMsIG1lYXN1cmUgPSAiYXVjIikKcmVkdWNlZF8xX2F1YyA8LSBhdWNAeS52YWx1ZXMKCiMjIHJlZHVjZWRfNApwcmVkczwtcHJlZGljdChyZWR1Y2VkXzQsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCnJhdGVzNDwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXM0LG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNCIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzQ8LXBlcmZvcm1hbmNlKHJhdGVzNCwgbWVhc3VyZSA9ICJhdWMiKQpyZWR1Y2VkXzRfYXVjIDwtIGF1YzRAeS52YWx1ZXMKCiMjIHJlZHVjZWRfNF8yCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF8yLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQpwbG90KHJvY19yZXN1bHQsIG1haW49IlJPQyBDdXJ2ZSBmb3IgcmVkdWNlZF80XzIiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM0XzI8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF8yX2F1YyA8LSBhdWM0XzJAeS52YWx1ZXMKCiMjIHJlZHVjZWRfNF8zCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF8zLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQpwbG90KHJvY19yZXN1bHQsIG1haW49IlJPQyBDdXJ2ZSBmb3IgcmVkdWNlZF80XzMiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF8zX2F1YyA8LSBhdWNAeS52YWx1ZXMKCiMjIHJlZHVjZWRfNF80X2xldiAKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzRfbGV2LG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQpwbG90KHJvY19yZXN1bHQsIG1haW49IlJPQyBDdXJ2ZSBmb3IgcmVkdWNlZF80XzRfbGV2IikKbGluZXMoeCA9IGMoMCwxKSwgeSA9IGMoMCwxKSwgY29sPSJyZWQiKQoKYXVjPC1wZXJmb3JtYW5jZShyYXRlcywgbWVhc3VyZSA9ICJhdWMiKQpyZWR1Y2VkXzRfNF9sZXZfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKIyMgcmVkdWNlZF80XzVfRERGRklUUyAKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzVfRERGRklUUyxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNF81X0RERkZJVFMiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF81X0RERkZJVFNfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKIyMgcmVkdWNlZF80XzZfbm9fc3BlY2lhbCAKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzZfbm9fc3BlY2lhbCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNF82X25vX3NwZWNpYWwiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF82X25vX3NwZWNpYWxfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKQVVDX3N1bW1hcnkgPC0gZGF0YS5mcmFtZSgncmVkdWNlZF8xJz1yZWR1Y2VkXzFfYXVjLAogICAgICAgICAgICAgICAgICAgICAgICAgICdyZWR1Y2VkXzQnPXJlZHVjZWRfNF9hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF8yJz1yZWR1Y2VkXzRfMl9hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF8zJz1yZWR1Y2VkXzRfM19hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF80X2xldic9cmVkdWNlZF80XzRfbGV2X2F1YywKICAgICAgICAgICAgICAgICAgICAgICAgICAncmVkdWNlZF80XzVfRERGRklUUyc9cmVkdWNlZF80XzVfRERGRklUU19hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF82X25vX3NwZWNpYWwnPXJlZHVjZWRfNF82X25vX3NwZWNpYWxfYXVjKQpjb2xuYW1lcyhBVUNfc3VtbWFyeSkgPC0gYygncmVkdWNlZF8xJywncmVkdWNlZF80JywncmVkdWNlZF80XzInLCdyZWR1Y2VkXzRfMycsJ3JlZHVjZWRfNF80X2xldicsJ3JlZHVjZWRfNF81X0RERkZJVFMnLCdyZWR1Y2VkXzRfNl9ub19zcGVjaWFsJykKCkFVQ19zdW1tYXJ5CmBgYAoKCgoKCiMjIFJ5YW4ncyBwYXJ0IHN0YXJ0cyBoZXJlCgojIyMgVGhlIGdvYWwgaXMgdG8gbWFrZSAzIG1vZGVsczogT25lIGZvciBqdXN0IHdoaXRlLCBvbmUgZm9yIGp1c3QgcmVkLCBhbmQgb25lIHdpdGggaW50ZXJhY3Rpb24gdGVybXMgd2l0aCB0aGUgdHlwZSBvZiB3aW5lLiAKCiMjIyMgQWZ0ZXIgdGhhdCwgdGhlIG1vZGVscyB3aWxsIGJlIHRyYWluZWQgb24gdGhlIGZpbHRlcmVkIGRhdGFzZXRzIGFuZCB0aGUgcmVzdWx0aW5nIHNjb3JlcyB3aWxsIGJlIGFkZGVkIHRvIHRoZSBldmFsdWF0aW9uIHN1bW1hcnkuCgoKIyMgUmVkIHdpbmUgb25seSBtb2RlbApgYGB7cn0KcmVnZnVsbF9SZWQ8LWdsbShjYXRfcXVhbGl0eX4uLCBmYW1pbHk9ImJpbm9taWFsIiwgZGF0YT10cmFpbl9SZWRfTm9UeXBlKQpyZWdudWxsX1JlZDwtZ2xtKGNhdF9xdWFsaXR5fjEsIGZhbWlseT0iYmlub21pYWwiLCBkYXRhPXRyYWluX1JlZF9Ob1R5cGUpCnN0ZXAocmVnbnVsbF9SZWQsIHNjb3BlPWxpc3QobG93ZXI9cmVnbnVsbF9SZWQsIHVwcGVyPXJlZ2Z1bGxfUmVkKSwgZGlyZWN0aW9uPSJmb3J3YXJkIikKCmBgYAoKClRoZSBtb2RlbCBsb29rcyBncmVhdCBhZnRlciB0aGUgZm93YXJkIHNlbGVjdGlvbiEgVGltZSB0byB0ZXN0IGFuZCBhZGQgdG8gdGhlIGV2YWx1YXRpb24gc3VtbWFyeS4KYGBge3J9Cm1vZGVsMV9SZWQ8LWdsbShmb3JtdWxhID0gY2F0X3F1YWxpdHkgfiBhbGNvaG9sICsgdm9sYXRpbGUuYWNpZGl0eSArIHRvdGFsLnN1bGZ1ci5kaW94aWRlICsgCiAgICBzdWxwaGF0ZXMgKyBmcmVlLnN1bGZ1ci5kaW94aWRlICsgY2hsb3JpZGVzLCBmYW1pbHkgPSAiYmlub21pYWwiLCAKICAgIGRhdGEgPSB0cmFpbl9SZWRfTm9UeXBlKQoKc3VtbWFyeShtb2RlbDFfUmVkKQoKCmBgYAoKYGBge3J9CgojI2V2YWx1YXRpbmcgbW9kZWwKbW9kZWwxX1JlZF9BSUNfdHJhaW4gPC0gbW9kZWwxX1JlZCRhaWMKIyNwcmVkaWN0ZWQgcXVhbGl0eSBmb3IgdGVzdCBkYXRhIGJhc2VkIG9uIHRyYWluaW5nIGRhdGEKdGVzdF9SZWRfTm9UeXBlPC1zdWJzZXQodGVzdCwgVHlwZSA9PSAwLCBzZWxlY3Q9LWMoVHlwZSkpCnByZWRzPC1wcmVkaWN0KG1vZGVsMV9SZWQsbmV3ZGF0YT10ZXN0X1JlZF9Ob1R5cGUsIHR5cGU9InJlc3BvbnNlIikKbW9kZWwxX1JlZF9lcnJvciA8LSB0YWJsZSh0ZXN0X1JlZF9Ob1R5cGUkY2F0X3F1YWxpdHksIHByZWRzPjAuNykKI0N1cnZlcwpldnVsYXRpb25fc3VtbWFyeV8xUiA8LSBkYXRhLmZyYW1lKAogIGF0dGVtcHQgPSAnbW9kZWwxX1JlZCcsCiAgQUlDID0gbW9kZWwxX1JlZF9BSUNfdHJhaW4sCiAgUFJFU1MgPSBnZXRfcHJlc3MobW9kZWwxX1JlZCksCiAgJ0ZhbHNlIHBvc2l0aXZlJyA9IHJvdW5kKG1vZGVsMV9SZWRfZXJyb3JbM10vKG1vZGVsMV9SZWRfZXJyb3JbMV0rbW9kZWwxX1JlZF9lcnJvclszXSksMyksCiAgJ0ZhbHNlIG5lZ2F0aXZlJyA9IHJvdW5kKG1vZGVsMV9SZWRfZXJyb3JbMl0vKG1vZGVsMV9SZWRfZXJyb3JbMl0rbW9kZWwxX1JlZF9lcnJvcls0XSksMyksCiAgJ0Vycm9yIFJhdGUnID0gcm91bmQoKG1vZGVsMV9SZWRfZXJyb3JbMl0rbW9kZWwxX1JlZF9lcnJvclszXSkvKG1vZGVsMV9SZWRfZXJyb3JbMV0rbW9kZWwxX1JlZF9lcnJvclsyXSttb2RlbDFfUmVkX2Vycm9yWzNdK21vZGVsMV9SZWRfZXJyb3JbNF0pLDMpCikKCmNvbXBhcmVfbW9kZWxzPC1yYmluZChldnVsYXRpb25fc3VtbWFyeVsxLF0sZXZ1bGF0aW9uX3N1bW1hcnlfMVIpCmNvbXBhcmVfbW9kZWxzCgpldnVsYXRpb25fc3VtbWFyeSA8LSByYmluZChldnVsYXRpb25fc3VtbWFyeSxldnVsYXRpb25fc3VtbWFyeV8xUikKZXZ1bGF0aW9uX3N1bW1hcnkKCmBgYApgYGB7cn0KIyBtb2RlbDFfUmVkCmxpYnJhcnkoUk9DUikKcHJlZHM8LXByZWRpY3QobW9kZWwxX1JlZCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9SZWQiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCm1vZGVsMV9SZWRfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKYGBgCgojIyBXaGl0ZSB3aW5lIG9ubHkgbW9kZWwKCmBgYHtyfQpyZWdmdWxsX1doaXRlPC1nbG0oY2F0X3F1YWxpdHl+LiwgZmFtaWx5PSJiaW5vbWlhbCIsIGRhdGE9dHJhaW5fV2hpdGVfTm9UeXBlKQpyZWdudWxsX1doaXRlPC1nbG0oY2F0X3F1YWxpdHl+MSxmYW1pbHk9ImJpbm9taWFsIiwgZGF0YT10cmFpbl9XaGl0ZV9Ob1R5cGUpCnN0ZXAocmVnbnVsbF9XaGl0ZSwgc2NvcGU9bGlzdChsb3dlcj1yZWdudWxsX1doaXRlLCB1cHBlcj1yZWdmdWxsX1doaXRlKSwgZGlyZWN0aW9uPSJmb3J3YXJkIikKYGBgCgoKVGhlIG1vZGVsIGxvb2tzIGdvb2QgYWZ0ZXIgdGhlIGZvd2FyZCBzZWxlY3Rpb24sIGJ1dCB0aGUgcHJlZGljdG9yIGZpeGVkLmFjaWRpdHkgY2FuIGJlIHJlbW92ZWQuIFRoZSBkZW5zaXR5IFZJRiBpcyBhYm92ZSB0ZW4sIGJ1dCBqc3V0IGJhcmVseS4gRm9yIG5vdywgaXQgd2lsbCBiZSBsZWZ0IGluLiBUaW1lIHRvIHRlc3QgYW5kIGFkZCB0byB0aGUgZXZhbHVhdGlvbiBzdW1tYXJ5LgpgYGB7cn0KbW9kZWwxX1doaXRlPC1nbG0oZm9ybXVsYSA9IGNhdF9xdWFsaXR5IH4gYWxjb2hvbCArIHZvbGF0aWxlLmFjaWRpdHkgKyByZXNpZHVhbC5zdWdhciArIAogICAgZml4ZWQuYWNpZGl0eSArIHN1bHBoYXRlcyArIGZyZWUuc3VsZnVyLmRpb3hpZGUgKyBkZW5zaXR5ICsgCiAgICBwSCwgZmFtaWx5ID0gImJpbm9taWFsIiwgZGF0YSA9IHRyYWluX1doaXRlX05vVHlwZSkKCgpzdW1tYXJ5KG1vZGVsMV9XaGl0ZSkKCgptb2RlbDFfV2hpdGU8LWdsbShmb3JtdWxhID0gY2F0X3F1YWxpdHkgfiBhbGNvaG9sICsgdm9sYXRpbGUuYWNpZGl0eSArIHJlc2lkdWFsLnN1Z2FyICsgc3VscGhhdGVzICsgZnJlZS5zdWxmdXIuZGlveGlkZSArIGRlbnNpdHkgKyAKICAgIHBILCBmYW1pbHkgPSAiYmlub21pYWwiLCBkYXRhID0gdHJhaW5fV2hpdGVfTm9UeXBlKQoKc3VtbWFyeShtb2RlbDFfV2hpdGUpCgoKYGBgCgoKYGBge3J9CgojI2V2YWx1YXRpbmcgbW9kZWwKbW9kZWwxX1doaXRlX0FJQ190cmFpbiA8LSBtb2RlbDFfV2hpdGUkYWljCiMjcHJlZGljdGVkIHF1YWxpdHkgZm9yIHRlc3QgZGF0YSBiYXNlZCBvbiB0cmFpbmluZyBkYXRhCnRlc3RfV2hpdGVfTm9UeXBlPC1zdWJzZXQodGVzdCwgVHlwZSA9PSAxLCBzZWxlY3Q9LWMoVHlwZSkpCnByZWRzPC1wcmVkaWN0KG1vZGVsMV9XaGl0ZSxuZXdkYXRhPXRlc3RfV2hpdGVfTm9UeXBlLCB0eXBlPSJyZXNwb25zZSIpCm1vZGVsMV9XaGl0ZV9lcnJvciA8LSB0YWJsZSh0ZXN0X1doaXRlX05vVHlwZSRjYXRfcXVhbGl0eSwgcHJlZHM+MC43KQojQ3VydmVzCmV2dWxhdGlvbl9zdW1tYXJ5XzFXIDwtIGRhdGEuZnJhbWUoCiAgYXR0ZW1wdCA9ICdtb2RlbDFfV2hpdGUnLAogIEFJQyA9IG1vZGVsMV9XaGl0ZV9BSUNfdHJhaW4sCiAgUFJFU1MgPSBnZXRfcHJlc3MobW9kZWwxX1doaXRlKSwKICAnRmFsc2UgcG9zaXRpdmUnID0gcm91bmQobW9kZWwxX1doaXRlX2Vycm9yWzNdLyhtb2RlbDFfV2hpdGVfZXJyb3JbMV0rbW9kZWwxX1doaXRlX2Vycm9yWzNdKSwzKSwKICAnRmFsc2UgbmVnYXRpdmUnID0gcm91bmQobW9kZWwxX1doaXRlX2Vycm9yWzJdLyhtb2RlbDFfV2hpdGVfZXJyb3JbMl0rbW9kZWwxX1doaXRlX2Vycm9yWzRdKSwzKSwKICAnRXJyb3IgUmF0ZScgPSByb3VuZCgobW9kZWwxX1doaXRlX2Vycm9yWzJdK21vZGVsMV9XaGl0ZV9lcnJvclszXSkvKG1vZGVsMV9XaGl0ZV9lcnJvclsxXSttb2RlbDFfV2hpdGVfZXJyb3JbMl0rbW9kZWwxX1doaXRlX2Vycm9yWzNdK21vZGVsMV9XaGl0ZV9lcnJvcls0XSksMykKKQoKZXZ1bGF0aW9uX3N1bW1hcnkgPC0gcmJpbmQoZXZ1bGF0aW9uX3N1bW1hcnksZXZ1bGF0aW9uX3N1bW1hcnlfMVcpCmV2dWxhdGlvbl9zdW1tYXJ5CgoKY29tcGFyZV9tb2RlbHM8LXJiaW5kKGNvbXBhcmVfbW9kZWxzLGV2dWxhdGlvbl9zdW1tYXJ5XzFXKQpjb21wYXJlX21vZGVscwoKYGBgCgoKCiMjIFRoZSBtb2RlbCB3aXRoIGludGVyYWN0aW9uIHRlcm1zCgpgYGB7cn0KcmVnZnVsbF9pbnQ8LWdsbShjYXRfcXVhbGl0eX4uKlR5cGUsIGZhbWlseT0iYmlub21pYWwiLCBkYXRhPXRyYWluKQpyZWdudWxsX2ludDwtZ2xtKGNhdF9xdWFsaXR5fjEsZmFtaWx5PSJiaW5vbWlhbCIsIGRhdGE9dHJhaW4pCnN0ZXAocmVnbnVsbF9pbnQsIHNjb3BlPWxpc3QobG93ZXI9cmVnbnVsbF9pbnQsIHVwcGVyPXJlZ2Z1bGxfaW50KSwgZGlyZWN0aW9uPSJmb3J3YXJkIikKYGBgCgoKVGhlIGZvcndhcmQgc3RlcCBwcm9jZXNzIGRyb3BwZWQgKyBzdWxwaGF0ZXM6VHlwZSwgZml4ZWQuYWNpZGl0eSwgYWxjb2hvbDpUeXBlLCBhbmQgY2l0cmljLmFjaWQgIEJ5IHRoZSBoaWVyYXJjaGljYWwgcHJpbmNpcGxlLCB0aGUgdHdvIG5vbi1pbnRlcmFjdGl2ZSB0ZXJtcyBuZWVkIHRvIGJlIGFkZGVkIGJhY2sgYmVjYXVzZSB0aGVpciBoYXZlIGludGVyYWN0aW9uIHRlcm1zIGFyZSBpbiB0aGUgbW9kZWwuCmBgYHtyfQogbW9kZWwxX2ludDwtZ2xtKGZvcm11bGEgPSBjYXRfcXVhbGl0eSB+IGFsY29ob2wgKyB2b2xhdGlsZS5hY2lkaXR5ICsgZGVuc2l0eSArIAogICAgc3VscGhhdGVzICsgcmVzaWR1YWwuc3VnYXIgKyAgVHlwZSArIHRvdGFsLnN1bGZ1ci5kaW94aWRlICsgCiAgICBmcmVlLnN1bGZ1ci5kaW94aWRlICsgcEggKyBjaGxvcmlkZXMgKyB2b2xhdGlsZS5hY2lkaXR5OlR5cGUgKyAKICAgIFR5cGU6dG90YWwuc3VsZnVyLmRpb3hpZGUgKyBkZW5zaXR5OlR5cGUgKyByZXNpZHVhbC5zdWdhcjpUeXBlICsgCiAgICBUeXBlOnBIICsgVHlwZTpjaGxvcmlkZXMgKyBUeXBlOmZyZWUuc3VsZnVyLmRpb3hpZGUsIGZhbWlseSA9ICJiaW5vbWlhbCIsIAogICAgZGF0YSA9IHRyYWluKQpzdW1tYXJ5KG1vZGVsMV9pbnQpCgpgYGAKCgpgYGB7cn0KCiMjZXZhbHVhdGluZyBtb2RlbAptb2RlbDFfaW50X0FJQ190cmFpbiA8LSBtb2RlbDFfaW50JGFpYwojI3ByZWRpY3RlZCBxdWFsaXR5IGZvciB0ZXN0IGRhdGEgYmFzZWQgb24gdHJhaW5pbmcgZGF0YQpwcmVkczwtcHJlZGljdChtb2RlbDFfaW50LG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQptb2RlbDFfaW50X2Vycm9yIDwtIHRhYmxlKHRlc3QkY2F0X3F1YWxpdHksIHByZWRzPjAuNykKI0N1cnZlcwpldnVsYXRpb25fc3VtbWFyeV8xaW50IDwtIGRhdGEuZnJhbWUoCiAgYXR0ZW1wdCA9ICdtb2RlbDFfaW50JywKICBBSUMgPSBtb2RlbDFfaW50X0FJQ190cmFpbiwKICBQUkVTUyA9IGdldF9wcmVzcyhtb2RlbDFfaW50KSwKICAnRmFsc2UgcG9zaXRpdmUnID0gcm91bmQobW9kZWwxX2ludF9lcnJvclszXS8obW9kZWwxX2ludF9lcnJvclsxXSttb2RlbDFfaW50X2Vycm9yWzNdKSwzKSwKICAnRmFsc2UgbmVnYXRpdmUnID0gcm91bmQobW9kZWwxX2ludF9lcnJvclsyXS8obW9kZWwxX2ludF9lcnJvclsyXSttb2RlbDFfaW50X2Vycm9yWzRdKSwzKSwKICAnRXJyb3IgUmF0ZScgPSByb3VuZCgobW9kZWwxX2ludF9lcnJvclsyXSttb2RlbDFfaW50X2Vycm9yWzNdKS8obW9kZWwxX2ludF9lcnJvclsxXSttb2RlbDFfaW50X2Vycm9yWzJdK21vZGVsMV9pbnRfZXJyb3JbM10rbW9kZWwxX2ludF9lcnJvcls0XSksMykKKQoKZXZ1bGF0aW9uX3N1bW1hcnkgPC0gcmJpbmQoZXZ1bGF0aW9uX3N1bW1hcnksZXZ1bGF0aW9uX3N1bW1hcnlfMWludCkKZXZ1bGF0aW9uX3N1bW1hcnkKCmNvbXBhcmVfbW9kZWxzPC1yYmluZChjb21wYXJlX21vZGVscyxldnVsYXRpb25fc3VtbWFyeV8xaW50KQpjb21wYXJlX21vZGVscwoKYGBgCgpgYGB7cn0KY29tcGFyZV9tb2RlbHM8LWNvbXBhcmVfbW9kZWxzJT4lIAogIHJlbmFtZSgKICAgIE1vZGVsID0gYXR0ZW1wdAogICAgKQoKYGBgCgoKYGBge3J9CmxpYnJhcnkoZGF0YS50YWJsZSkKbGlicmFyeShkcGx5cikKbGlicmFyeShmb3JtYXR0YWJsZSkKbGlicmFyeSh0aWR5cikKY3VzdG9tR3JlZW4wID0gIiNEZUY3RTkiCgpjdXN0b21HcmVlbiA9ICIjNzFDQTk3IgoKY3VzdG9tUmVkID0gIiNmZjdmN2YiCgpgYGAKCgoKCgoKCgoKCgoKCkNyZWF0aW5nIHRoZSBST0MgY3VydmVzIGFuZCBBVUMgZm9yIHRoZSAzIG5ldyBtb2RlbHMuCmBgYHtyfQojIG1vZGVsMV9SZWQKcHJlZHM8LXByZWRpY3QobW9kZWwxX1JlZCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9SZWQiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCm1vZGVsMV9SZWRfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKCiMgbW9kZWwxX1doaXRlCnByZWRzPC1wcmVkaWN0KG1vZGVsMV9XaGl0ZSxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9XaGl0ZSIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzwtcGVyZm9ybWFuY2UocmF0ZXMsIG1lYXN1cmUgPSAiYXVjIikKbW9kZWwxX1doaXRlX2F1YyA8LSBhdWNAeS52YWx1ZXMKCgojIG1vZGVsMV9pbnQKcHJlZHM8LXByZWRpY3QobW9kZWwxX2ludCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9pbnQiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCm1vZGVsMV9pbnRfYXVjIDwtIGF1Y0B5LnZhbHVlcwpgYGAKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKIyMgVGhpcyBpcyB0aGUgb25lIGxpbmVycyB0aGF0IHJ1biB0aGUgdGFibGVzIGFuZCBmaWd1cmVzIQoKCgoKCmBgYHtyfQoKIyNjcmVhdGUgaGVhdCBtYXAgQ29uc29saWRhdGVkCmdncGxvdChkYXRhID0gbWVsdGVkX2Nvcl90cmFpbiwgYWVzKHg9VmFyMSwgeT1WYXIyLCBmaWxsPXZhbHVlKSkgKyAKICBnZW9tX3RpbGUoY29sb3IgPSAid2hpdGUiKSsKICBzY2FsZV9maWxsX2dyYWRpZW50Mihsb3cgPSAiYmx1ZSIsIGhpZ2ggPSAicmVkIiwgbWlkID0gIndoaXRlIiwgbWlkcG9pbnQgPSAwLCBsaW1pdCA9IGMoLTEsMSksIHNwYWNlID0gIkxhYiIsIG5hbWU9IlBlYXJzb25cbkNvcnJlbGF0aW9uIikgKwogIHRoZW1lX21pbmltYWwoKSsgCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgdmp1c3QgPSAxLCBzaXplID0gMTIsIGhqdXN0ID0gMSkpKwogIGNvb3JkX2ZpeGVkKCkrCiAgbGFicyh0aXRsZSA9ICdDb25zb2xpZGF0ZWQgKEJvdGggUmVkIGFuZCBXaGl0ZSknKQoKYGBgCgpgYGB7cn0KIyNjcmVhdGUgaGVhdCBtYXAgV2hpdGUKZ2dwbG90KGRhdGEgPSBtZWx0ZWRfY29yX3RyYWluX3doaXRlLCBhZXMoeD1WYXIxLCB5PVZhcjIsIGZpbGw9dmFsdWUpKSArCiAgZ2VvbV90aWxlKGNvbG9yID0gIndoaXRlIikrCiAgc2NhbGVfZmlsbF9ncmFkaWVudDIobG93ID0gImJsdWUiLCBoaWdoID0gInJlZCIsIG1pZCA9ICJ3aGl0ZSIsIG1pZHBvaW50ID0gMCwgbGltaXQgPSBjKC0xLDEpLCBzcGFjZSA9ICJMYWIiLCBuYW1lPSJQZWFyc29uXG5Db3JyZWxhdGlvbiIpICsKICB0aGVtZV9taW5pbWFsKCkrCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgdmp1c3QgPSAxLCBzaXplID0gMTIsIGhqdXN0ID0gMSkpKwogIGNvb3JkX2ZpeGVkKCkrCiAgbGFicyh0aXRsZSA9ICdXaGl0ZSBXaW5lJykKCmBgYAoKYGBge3J9CiMjY3JlYXRlIGhlYXQgbWFwIFJlZApnZ3Bsb3QoZGF0YSA9IG1lbHRlZF9jb3JfdHJhaW5fUmVkLCBhZXMoeD1WYXIxLCB5PVZhcjIsIGZpbGw9dmFsdWUpKSArCiAgZ2VvbV90aWxlKGNvbG9yID0gIndoaXRlIikrCiAgc2NhbGVfZmlsbF9ncmFkaWVudDIobG93ID0gImJsdWUiLCBoaWdoID0gInJlZCIsIG1pZCA9ICJ3aGl0ZSIsIG1pZHBvaW50ID0gMCwgbGltaXQgPSBjKC0xLDEpLCBzcGFjZSA9ICJMYWIiLCBuYW1lPSJQZWFyc29uXG5Db3JyZWxhdGlvbiIpICsKICB0aGVtZV9taW5pbWFsKCkrCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgdmp1c3QgPSAxLCBzaXplID0gMTIsIGhqdXN0ID0gMSkpKwogIGNvb3JkX2ZpeGVkKCkrCiAgbGFicyh0aXRsZSA9ICdSZWQgV2luZScpCgpgYGAKCgoKCgpgYGB7cn0KCgpnZ3Bsb3QodHJhaW5fd2l0aF9xdWFsLCBtYXBwaW5nID0gYWVzKHg9cXVhbGl0eSwgZmlsbD1UeXBlKSkrCiAgZ2VvbV9oaXN0b2dyYW0oYmlud2lkdGg9MSwgYWxwaGE9LjQsIHBvc2l0aW9uPSJpZGVudGl0eSIsIGNvbG9yPSJibGFjayIpKwogIGdlb21fdmxpbmUoYWVzKHhpbnRlcmNlcHQ9NS41LCBjb2xvcj0icmVkIiksCiAgICAgICAgICAgICBsaW5ldHlwZT0iZGFzaGVkIikrCiAgc2NhbGVfY29sb3JfbWFudWFsKG5hbWUgPSAiQ3V0IE9mZiIsIHZhbHVlcyA9IGMoInJlZCIpKSsKICBsYWJzKHg9IlF1YWxpdHkiLAogICAgICAgeT0iRnJlcXVlbmN5IiwKICAgICAgIHRpdGxlPSJEaXN0cmlidXRpb24gb2YgUXVhbGl0eSBSYXRpbmcgYnkgV2luZSBUeXBlIikKYGBgCgoKClRoaXMgaXMgdGhlIHRhYmxlIGZvciBzaG93aW5nIHRoZSBldmFsdWF0aW9uIGZvciB0aGUgZmlyc3QgbW9kZWwKYGBge3J9CmZvcm1hdHRhYmxlKGV2dWxhdGlvbl9zdW1tYXJ5WzEsXSkKYGBgCgoKCmBgYHtyfQpzdW1tYXJ5KG1vZGVsMV9SZWQpCmBgYAoKYGBge3J9CnN1bW1hcnkobW9kZWwxX1doaXRlKQpgYGAKCgpgYGB7cn0Kc3VtbWFyeShtb2RlbDFfaW50KQpgYGAKCgoKClRhYmxlIHJpZ2h0IGFib3ZlIHRoZSAiQmVzdCBQb3NzaWJsZSBNb2RlbCAoUmVkdWNlZF80KSIgc2VjdGlvbi4KYGBge3J9CmZvcm1hdHRhYmxlKGNvbXBhcmVfbW9kZWxzLGFsaWduID1jKCJsIiwiYyIsICJjIiwgImMiLCAiYyIsICJyIikpCgpgYGAKCgoKClRoaXMgaXMgdGhlIHRhYmxlIGZvciBzaG93aW5nIHRoZSBiZXN0IG1vZGVscyAodG9wIGZpdmUpCmBgYHtyfQpmb3JtYXR0YWJsZShyZXMuYmVzdC5sb2dpc3RpYyRCZXN0TW9kZWxzKQpgYGAKCgpBZGQgUk9DIGZvciByZWR1Y2VkXzEsIG1vZGVsMV9SZWQsIG1vZGVsMV9XaGl0ZSwgbW9kZWwxX2ludAoKYGBge3J9CgpwcmVkczwtcHJlZGljdChyZWR1Y2VkXzEsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCnJhdGVzPC1wcmVkaWN0aW9uKHByZWRzLCB0ZXN0JGNhdF9xdWFsaXR5KQpyb2NfcmVzdWx0PC1wZXJmb3JtYW5jZShyYXRlcyxtZWFzdXJlPSJ0cHIiLCB4Lm1lYXN1cmU9ImZwciIpCnBsb3Qocm9jX3Jlc3VsdCwgbWFpbj0iUk9DIEN1cnZlIGZvciByZWR1Y2VkXzEiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfMV9hdWMgPC0gYXVjQHkudmFsdWVzCgpgYGAKCgpgYGB7cn0KIyBtb2RlbDFfUmVkCnByZWRzPC1wcmVkaWN0KG1vZGVsMV9SZWQsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCnJhdGVzPC1wcmVkaWN0aW9uKHByZWRzLCB0ZXN0JGNhdF9xdWFsaXR5KQpyb2NfcmVzdWx0PC1wZXJmb3JtYW5jZShyYXRlcyxtZWFzdXJlPSJ0cHIiLCB4Lm1lYXN1cmU9ImZwciIpCnBsb3Qocm9jX3Jlc3VsdCwgbWFpbj0iUk9DIEN1cnZlIGZvciBtb2RlbDFfUmVkIikKbGluZXMoeCA9IGMoMCwxKSwgeSA9IGMoMCwxKSwgY29sPSJyZWQiKQoKYXVjPC1wZXJmb3JtYW5jZShyYXRlcywgbWVhc3VyZSA9ICJhdWMiKQptb2RlbDFfUmVkX2F1YyA8LSBhdWNAeS52YWx1ZXMKCmBgYAoKYGBge3J9CgojIG1vZGVsMV9XaGl0ZQpwcmVkczwtcHJlZGljdChtb2RlbDFfV2hpdGUsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCnJhdGVzPC1wcmVkaWN0aW9uKHByZWRzLCB0ZXN0JGNhdF9xdWFsaXR5KQpyb2NfcmVzdWx0PC1wZXJmb3JtYW5jZShyYXRlcyxtZWFzdXJlPSJ0cHIiLCB4Lm1lYXN1cmU9ImZwciIpCnBsb3Qocm9jX3Jlc3VsdCwgbWFpbj0iUk9DIEN1cnZlIGZvciBtb2RlbDFfV2hpdGUiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCm1vZGVsMV9XaGl0ZV9hdWMgPC0gYXVjQHkudmFsdWVzCgpgYGAKCmBgYHtyfQoKIyBtb2RlbDFfaW50CnByZWRzPC1wcmVkaWN0KG1vZGVsMV9pbnQsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCnJhdGVzPC1wcmVkaWN0aW9uKHByZWRzLCB0ZXN0JGNhdF9xdWFsaXR5KQpyb2NfcmVzdWx0PC1wZXJmb3JtYW5jZShyYXRlcyxtZWFzdXJlPSJ0cHIiLCB4Lm1lYXN1cmU9ImZwciIpCnBsb3Qocm9jX3Jlc3VsdCwgbWFpbj0iUk9DIEN1cnZlIGZvciBtb2RlbDFfaW50IikKbGluZXMoeCA9IGMoMCwxKSwgeSA9IGMoMCwxKSwgY29sPSJyZWQiKQoKYXVjPC1wZXJmb3JtYW5jZShyYXRlcywgbWVhc3VyZSA9ICJhdWMiKQptb2RlbDFfaW50X2F1YyA8LSBhdWNAeS52YWx1ZXMKYGBgCgoKVGhpcyBpcyB0aGUgdGFibGUgZm9yIHNob3dpbmcgdGhlIGJlc3QgbW9kZWxzICh0b3AgZml2ZSkKYGBge3J9CmZvcm1hdHRhYmxlKGV2dWxhdGlvbl9zdW1tYXJ5WzIsXSkKYGBgCgoKVGhpcyBpcyB0aGUgdGFibGUgZm9yIHJlZHVjZWRfNCBWSUYuCmBgYHtyfQpmb3JtYXR0YWJsZShkYXRhLmZyYW1lKHJlZ180X1ZJRl90ZXN0KSwgYWxpZ24gPWMoImwiLCJyIikpCmBgYAoKVGhpcyBpcyB0aGUgbmV4dCBWSUYgcGxvdCBpbiB0aGUgcmVwb3J0CmBgYHtyfQpmb3JtYXR0YWJsZShkYXRhLmZyYW1lKHJlZ180XzJfVklGX3Rlc3QpLCBhbGlnbiA9YygibCIsInIiKSkKYGBgCgoKVGhlIHRhYmxlIGJlbG93IHRoYXQuIEl0IGlzIHRoZSBldmFsdWF0aW9uIHN1bW1hcnkgZm9yIHJlZHVjZWRfNF8yCmBgYHtyfQpmb3JtYXR0YWJsZShldnVsYXRpb25fc3VtbWFyeVszLF0pCmBgYAoKCmV2YWx1YXRpb24gc3VtbWFyeSBmb3IgdGhlIG91dGxpZXIvbGV2ZXJhZ2UvZXRjLgpgYGB7cn0KZm9ybWF0dGFibGUoZXZ1bGF0aW9uX3N1bW1hcnlbNDo3LF0pCmBgYAoKCgphZGQgcm9jIGN1cnZlcyBmb3IgdGhlc2UgZm91ci4KCmBgYHtyfQoKCiMjIHJlZHVjZWRfNApwcmVkczwtcHJlZGljdChyZWR1Y2VkXzQsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCnJhdGVzNDwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXM0LG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNCIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzQ8LXBlcmZvcm1hbmNlKHJhdGVzNCwgbWVhc3VyZSA9ICJhdWMiKQpyZWR1Y2VkXzRfYXVjIDwtIGF1YzRAeS52YWx1ZXMKCmBgYAoKYGBge3J9CgojIyByZWR1Y2VkXzRfMgpwcmVkczwtcHJlZGljdChyZWR1Y2VkXzRfMixuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNF8yIikKbGluZXMoeCA9IGMoMCwxKSwgeSA9IGMoMCwxKSwgY29sPSJyZWQiKQoKYXVjNF8yPC1wZXJmb3JtYW5jZShyYXRlcywgbWVhc3VyZSA9ICJhdWMiKQpyZWR1Y2VkXzRfMl9hdWMgPC0gYXVjNF8yQHkudmFsdWVzCgpgYGAKCgpgYGB7cn0KCiMjIHJlZHVjZWRfNF8zCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF8zLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQpwbG90KHJvY19yZXN1bHQsIG1haW49IlJPQyBDdXJ2ZSBmb3IgcmVkdWNlZF80XzMiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF8zX2F1YyA8LSBhdWNAeS52YWx1ZXMKCgpgYGAKCgpgYGB7cn0KIyMgcmVkdWNlZF80XzRfbGV2IApwcmVkczwtcHJlZGljdChyZWR1Y2VkXzRfNF9sZXYsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCnJhdGVzPC1wcmVkaWN0aW9uKHByZWRzLCB0ZXN0JGNhdF9xdWFsaXR5KQpyb2NfcmVzdWx0PC1wZXJmb3JtYW5jZShyYXRlcyxtZWFzdXJlPSJ0cHIiLCB4Lm1lYXN1cmU9ImZwciIpCnBsb3Qocm9jX3Jlc3VsdCwgbWFpbj0iUk9DIEN1cnZlIGZvciByZWR1Y2VkXzRfNF9sZXYiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF80X2xldl9hdWMgPC0gYXVjQHkudmFsdWVzCgpgYGAKCmBgYHtyfQoKIyMgcmVkdWNlZF80XzVfRERGRklUUyAKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzVfRERGRklUUyxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNF81X0RERkZJVFMiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF81X0RERkZJVFNfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKYGBgCgoKCgpgYGB7cn0KCiMjIHJlZHVjZWRfNF82X25vX3NwZWNpYWwgCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF82X25vX3NwZWNpYWwsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCnJhdGVzPC1wcmVkaWN0aW9uKHByZWRzLCB0ZXN0JGNhdF9xdWFsaXR5KQpyb2NfcmVzdWx0PC1wZXJmb3JtYW5jZShyYXRlcyxtZWFzdXJlPSJ0cHIiLCB4Lm1lYXN1cmU9ImZwciIpCnBsb3Qocm9jX3Jlc3VsdCwgbWFpbj0iUk9DIEN1cnZlIGZvciByZWR1Y2VkXzRfNl9ub19zcGVjaWFsIikKbGluZXMoeCA9IGMoMCwxKSwgeSA9IGMoMCwxKSwgY29sPSJyZWQiKQoKYXVjPC1wZXJmb3JtYW5jZShyYXRlcywgbWVhc3VyZSA9ICJhdWMiKQpyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2F1YyA8LSBhdWNAeS52YWx1ZXMKCmBgYAoKYGBge3J9CkFVQ19zdW1tYXJ5IDwtIGRhdGEuZnJhbWUoJ3JlZHVjZWRfMSc9cmVkdWNlZF8xX2F1YywKICAgICAgICAgICAgICAgICAgICAgICAgICAncmVkdWNlZF80Jz1yZWR1Y2VkXzRfYXVjLAogICAgICAgICAgICAgICAgICAgICAgICAgICdyZWR1Y2VkXzRfMic9cmVkdWNlZF80XzJfYXVjLAogICAgICAgICAgICAgICAgICAgICAgICAgICdyZWR1Y2VkXzRfMyc9cmVkdWNlZF80XzNfYXVjLAogICAgICAgICAgICAgICAgICAgICAgICAgICdyZWR1Y2VkXzRfNF9sZXYnPXJlZHVjZWRfNF80X2xldl9hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF81X0RERkZJVFMnPXJlZHVjZWRfNF81X0RERkZJVFNfYXVjLAogICAgICAgICAgICAgICAgICAgICAgICAgICdyZWR1Y2VkXzRfNl9ub19zcGVjaWFsJz1yZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2F1YykKY29sbmFtZXMoQVVDX3N1bW1hcnkpIDwtIGMoJ3JlZHVjZWRfMScsJ3JlZHVjZWRfNCcsJ3JlZHVjZWRfNF8yJywncmVkdWNlZF80XzMnLCdyZWR1Y2VkXzRfNF9sZXYnLCdyZWR1Y2VkXzRfNV9EREZGSVRTJywncmVkdWNlZF80XzZfbm9fc3BlY2lhbCcpCgpBVUNfc3VtbWFyeQpgYGAKCgoKCgojCgoK